From a56601c98c6e3ade398f7bf491d0e4be50ee4817 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 13:26:21 +0000 Subject: [PATCH 001/225] CI for 314/314t --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 4c7599a6..89f210be 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp39, cp310, cp311, cp312, cp313 ] + python: [ cp39, cp310, cp311, cp312, cp313, cp314, cp314t ] platform: - { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } diff --git a/pyproject.toml b/pyproject.toml index 6291b811..914444f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,9 +46,9 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "ipython", # used in duckdb.query_graph "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() - "pandas", # used for pandas dataframes all over the place - "pyarrow", # used for pyarrow support - "adbc_driver_manager", # for the adbc driver (TODO: this should live under the duckdb package) + "pandas; python_version < '3.14'", # used for pandas dataframes all over the place + "pyarrow; python_version < '3.14'", # used for pyarrow support + "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package) ] ###################################################################################################### @@ -229,20 +229,20 @@ test = [ # dependencies used for running tests "pytest-timeout", "mypy", "coverage", - "gcovr", + "gcovr; python_version < '3.14'", "gcsfs", "packaging", - "polars", + "polars; python_version < '3.14'", "psutil", "py4j", "pyotp", - "pyspark", + "pyspark; python_version < '3.14'", "pytz", "requests", "urllib3", "fsspec>=2022.11.0", - "pandas>=2.0.0", - "pyarrow>=18.0.0", + "pandas>=2.0.0; python_version < '3.14'", + "pyarrow>=18.0.0; python_version < '3.14'", "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'", @@ -256,10 +256,10 @@ scripts = [ # dependencies used for running scripts "ipython", "ipywidgets", "numpy", - "pandas", + "pandas; python_version < '3.14'", "pcpp", - "polars", - "pyarrow", + "polars; python_version < '3.14'", + "pyarrow; python_version < '3.14'", "pytz" ] pypi = [ # dependencies used by the pypi cleanup script From dbb374869e7902ac4d3188bdfb895b7c77d4776d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 13:28:08 +0000 Subject: [PATCH 002/225] Chore: CI - Disable everything but 314 for now. --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 89f210be..14c36937 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp39, cp310, cp311, cp312, cp313, cp314, cp314t ] + python: [ cp314, cp314t ] # cp39, cp310, cp311, cp312, cp313, platform: - { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } From 22e3a13c20209d0ff3b28809780c7f475ad8b2d6 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 13:31:35 +0000 Subject: [PATCH 003/225] bump uv version --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 14c36937..a35fae2f 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -79,7 +79,7 @@ jobs: # Install Astral UV, which will be used as build-frontend for cibuildwheel - uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" enable-cache: false cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} From e652913b429d45f04f6975522d4ac3f4544e3308 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 13:32:30 +0000 Subject: [PATCH 004/225] cache too --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a35fae2f..fec4b777 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -80,7 +80,7 @@ jobs: - uses: astral-sh/setup-uv@v6 with: version: "0.8.16" - enable-cache: false + enable-cache: true cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels From 64ab8f89288c9ad355f93e391b2cc2c1c879efb1 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 13:37:38 +0000 Subject: [PATCH 005/225] Chore: CI - reduce matrix and enable experimental for 314 --- .github/workflows/packaging_wheels.yml | 10 +++++----- pyproject.toml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index fec4b777..8a9a54c4 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -32,12 +32,12 @@ jobs: matrix: python: [ cp314, cp314t ] # cp39, cp310, cp311, cp312, cp313, platform: - - { os: windows-2025, arch: amd64, cibw_system: win } + #- { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - - { os: macos-15, arch: arm64, cibw_system: macosx } - - { os: macos-15, arch: universal2, cibw_system: macosx } - - { os: macos-13, arch: x86_64, cibw_system: macosx } + #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + #- { os: macos-15, arch: arm64, cibw_system: macosx } + #- { os: macos-15, arch: universal2, cibw_system: macosx } + #- { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: diff --git a/pyproject.toml b/pyproject.toml index 914444f7..c3125474 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -379,6 +379,7 @@ manylinux-x86_64-image = "manylinux_2_28" manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" manylinux-pypy_aarch64-image = "manylinux_2_28" +enable = ["experimental"] [tool.cibuildwheel.linux] before-build = ["yum install -y ccache"] From e8ec78a1f0cafe15e8b31e11fe2e5a858ec1f738 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 13:45:24 +0000 Subject: [PATCH 006/225] enable the right groups --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c3125474..7fe41938 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -379,7 +379,7 @@ manylinux-x86_64-image = "manylinux_2_28" manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" manylinux-pypy_aarch64-image = "manylinux_2_28" -enable = ["experimental"] +enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] before-build = ["yum install -y ccache"] From b1a52c038e9469d99073a5526e35bf186aaf8daf Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 13:49:54 +0000 Subject: [PATCH 007/225] for now, make sure uv is using the right python version using python-version. --- .github/workflows/packaging_wheels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 8a9a54c4..b90141e4 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -82,6 +82,9 @@ jobs: version: "0.8.16" enable-cache: true cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + python-version: ${{ + fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] + }} - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels uses: pypa/cibuildwheel@v3.0 From 0869d1cba1dfbac162ebaa7ca8ef177970100094 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 15:30:42 +0000 Subject: [PATCH 008/225] For testing, let's cache the build dir, for a single target --- .github/workflows/packaging_wheels.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b90141e4..af29a922 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp314, cp314t ] # cp39, cp310, cp311, cp312, cp313, + python: [ cp314 ] # cp39, cp310, cp311, cp312, cp313, , cp314t platform: #- { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } @@ -60,9 +60,17 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ inputs.duckdb-python-sha }} - fetch-depth: 0 + fetch-depth: 0 submodules: true - + - name: Cache Build Directory + uses: actions/cache@v4 + with: + path: | + build/ + wheelhouse/ + key: build-dir-${{ matrix.python }}-${{ runner.os }}-${{ hashFiles('external/duckdb/.git/HEAD', 'CMakeLists.txt') }} + restore-keys: | + build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -74,7 +82,7 @@ jobs: # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds - name: Set OVERRIDE_GIT_DESCRIBE if: ${{ inputs.set-version != '' }} - run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV + run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV # Install Astral UV, which will be used as build-frontend for cibuildwheel - uses: astral-sh/setup-uv@v6 @@ -91,7 +99,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} - name: Upload wheel uses: actions/upload-artifact@v4 with: From af8c43313554d26373ced5ef28a4391be3c4f820 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 15:35:27 +0000 Subject: [PATCH 009/225] Simplify cache --- .github/workflows/packaging_wheels.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index af29a922..224db43a 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,8 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run python -c "import duckdb + #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs steps: - name: Checkout DuckDB Python @@ -68,7 +69,7 @@ jobs: path: | build/ wheelhouse/ - key: build-dir-${{ matrix.python }}-${{ runner.os }}-${{ hashFiles('external/duckdb/.git/HEAD', 'CMakeLists.txt') }} + key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Checkout DuckDB From 780288b61d8cb780f3c6e8b76590e6217b1f64a5 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 15:37:40 +0000 Subject: [PATCH 010/225] Disable coverage --- .github/disabled/on_push.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/disabled/on_push.yml diff --git a/.github/disabled/on_push.yml b/.github/disabled/on_push.yml new file mode 100644 index 00000000..1a282d69 --- /dev/null +++ b/.github/disabled/on_push.yml @@ -0,0 +1,26 @@ +name: Tests and coverage on push +on: + push: + branches-ignore: + - main + - v*.*-* + paths-ignore: + - '**.md' + - 'LICENSE' + - '.editorconfig' + - 'scripts/**' + - '.github//**' + - '!.github/workflows/on_push.yml' + - '!.github/workflows/coverage.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Run coverage tests + uses: ./.github/workflows/coverage.yml + with: + git_ref: ${{ github.ref }} + testsuite: fast From 242ee3147a5491b135f5f542cf687528f7166359 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 15:37:50 +0000 Subject: [PATCH 011/225] Disable coverage --- .github/workflows/on_push.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .github/workflows/on_push.yml diff --git a/.github/workflows/on_push.yml b/.github/workflows/on_push.yml deleted file mode 100644 index 1a282d69..00000000 --- a/.github/workflows/on_push.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Tests and coverage on push -on: - push: - branches-ignore: - - main - - v*.*-* - paths-ignore: - - '**.md' - - 'LICENSE' - - '.editorconfig' - - 'scripts/**' - - '.github//**' - - '!.github/workflows/on_push.yml' - - '!.github/workflows/coverage.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - name: Run coverage tests - uses: ./.github/workflows/coverage.yml - with: - git_ref: ${{ github.ref }} - testsuite: fast From b112a71e248bea1bcab7290cf81e6ad7745d2056 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 16:06:25 +0000 Subject: [PATCH 012/225] chore: disable sdist and try to get ci happier --- .github/workflows/packaging.yml | 1 + .github/workflows/packaging_wheels.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 25cf3bdd..755b86c5 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -63,6 +63,7 @@ defaults: jobs: build_sdist: name: Build an sdist and determine versions + if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/packaging_sdist.yml with: testsuite: all diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 224db43a..eae318fe 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -51,7 +51,7 @@ jobs: CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && - uv pip install -r pylock.toml + uv pip install -r {project}/pylock.toml CIBW_TEST_COMMAND: > uv run python -c "import duckdb #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs From dac704ab36751bce3514d80190af3fbf16dbaa73 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 16:11:39 +0000 Subject: [PATCH 013/225] add ccache --- .github/workflows/packaging_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index eae318fe..fbea1149 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -69,6 +69,7 @@ jobs: path: | build/ wheelhouse/ + ~/.ccache key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- From e3e3c445122ce2d85f3ad92b895d95c668fc5e10 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 16:14:13 +0000 Subject: [PATCH 014/225] explicitly set ccache dir --- .github/workflows/packaging_wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index fbea1149..a3245678 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -47,6 +47,7 @@ jobs: - { minimal: true, platform: { arch: universal2 } } runs-on: ${{ matrix.platform.os }} env: + CCACHE_DIR: /tmp/ccache CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > @@ -69,7 +70,7 @@ jobs: path: | build/ wheelhouse/ - ~/.ccache + /tmp/ccache key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- From 214d335dc89ad84330ed8bbf1443ee801b85cefc Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 16:43:58 +0000 Subject: [PATCH 015/225] debug info: why failing --- .github/workflows/packaging_wheels.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a3245678..da13fbdc 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -51,8 +51,13 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && - uv pip install -r {project}/pylock.toml + pwd && ls -la && + python --version && uv --version && + echo "uv export" && + uv export --only-group test --no-emit-project --output-file pylock.toml --directory /project --verbose 2>&1 || echo "Export failed with exit code $?" && + echo "listing file" && + ls -la pylock.toml 2>/dev/null || echo "pylock.toml not created" && + echo "done" CIBW_TEST_COMMAND: > uv run python -c "import duckdb #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs From 5ba7743438bc4b172b15a7e2bb656b952df35ca0 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 17:01:40 +0000 Subject: [PATCH 016/225] always save cache, --- .github/workflows/packaging_wheels.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index da13fbdc..99dcc87e 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp314 ] # cp39, cp310, cp311, cp312, cp313, , cp314t + python: [ cp313 ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: #- { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } @@ -47,7 +47,6 @@ jobs: - { minimal: true, platform: { arch: universal2 } } runs-on: ${{ matrix.platform.os }} env: - CCACHE_DIR: /tmp/ccache CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > @@ -69,8 +68,8 @@ jobs: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - - name: Cache Build Directory - uses: actions/cache@v4 + - name: Restore Build Cache + uses: actions/cache/restore@v4 with: path: | build/ @@ -114,3 +113,12 @@ jobs: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl compression-level: 0 + - name: Save Build Cache + if: always() # Always save, even on failure + uses: actions/cache/save@v4 + with: + path: | + build/ + wheelhouse/ + /tmp/ccache + key: build-dir-${{ matrix.python }}-${{ runner.os }} \ No newline at end of file From b5769ea8abecc219db2a195d7f0bd6783ab852d4 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 17:11:03 +0000 Subject: [PATCH 017/225] back to 314t --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 99dcc87e..d291b7f6 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp313 ] # cp39, cp310, cp311, cp312, cp314, cp314t + python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: #- { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } From 4c4413cb705f378d15fc2fead4721f04da5a26fb Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 17:35:01 +0000 Subject: [PATCH 018/225] setting up ccache --- .github/workflows/packaging_wheels.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index d291b7f6..193be2da 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -68,13 +68,19 @@ jobs: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true + - name: Setup ccache + run: | + sudo apt-get update && sudo apt-get install -y ccache + mkdir -p /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + echo "CCACHE_DIR=/tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=2G" >> $GITHUB_ENV - name: Restore Build Cache uses: actions/cache/restore@v4 with: path: | build/ wheelhouse/ - /tmp/ccache + /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- @@ -120,5 +126,5 @@ jobs: path: | build/ wheelhouse/ - /tmp/ccache + /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} \ No newline at end of file From 45aab9c2553a9d275d99840d95ac81ae7dce5e90 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 17:38:47 +0000 Subject: [PATCH 019/225] ccache is already installed, just set env --- .github/workflows/packaging_wheels.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 193be2da..0db8ebf4 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -60,7 +60,10 @@ jobs: CIBW_TEST_COMMAND: > uv run python -c "import duckdb #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - + CCACHE_DIR: > + /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + CCACHE_MAXSIZE: 2G + steps: - name: Checkout DuckDB Python uses: actions/checkout@v4 @@ -68,12 +71,6 @@ jobs: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - - name: Setup ccache - run: | - sudo apt-get update && sudo apt-get install -y ccache - mkdir -p /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - echo "CCACHE_DIR=/tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=2G" >> $GITHUB_ENV - name: Restore Build Cache uses: actions/cache/restore@v4 with: @@ -84,6 +81,8 @@ jobs: key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- + - name: Show ccache stats + run: ccache -s - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} From 177aa6f2e80a13adf4894d62503e1c6a467c499d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 17:40:50 +0000 Subject: [PATCH 020/225] install ccache --- .github/workflows/packaging_wheels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 0db8ebf4..fd84ced1 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -71,6 +71,9 @@ jobs: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true + - name: Setup ccache + run: | + sudo apt-get update && sudo apt-get install -y ccache - name: Restore Build Cache uses: actions/cache/restore@v4 with: From e42b3d0281165da1fd736290a7b0c9f14296afff Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 17:52:00 +0000 Subject: [PATCH 021/225] fixing uv not using right version --- .github/workflows/packaging_wheels.yml | 22 +++++++++++++++++++--- pyproject.toml | 1 + 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index fd84ced1..c91be384 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -71,9 +71,11 @@ jobs: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - - name: Setup ccache - run: | - sudo apt-get update && sudo apt-get install -y ccache + - name: Install ccache + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: ccache + version: 1.0 - name: Restore Build Cache uses: actions/cache/restore@v4 with: @@ -109,6 +111,20 @@ jobs: fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] }} + - name: Verify Python version + run: | + expected="${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] }}" + actual=$(python --version | cut -d' ' -f2) + echo "Expected: $expected, Actual: $actual" + if [[ "$expected" == "3.14t" && "$actual" =~ ^3\.14\. ]]; then + echo "Python 3.14t version check passed" + elif [[ "$actual" =~ ^$expected\. ]]; then + echo "Python version check passed" + else + echo "Python version mismatch!" + exit 1 + fi + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels uses: pypa/cibuildwheel@v3.0 env: diff --git a/pyproject.toml b/pyproject.toml index 7fe41938..7b29c6fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,6 +204,7 @@ required-environments = [ # ... but do always resolve for all of them "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'x86_64'", "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'aarch64'", ] +prerelease = "allow" # for 3.14 # We just need pytorch for tests, wihtout GPU acceleration. PyPI doesn't host a cpu-only version for Linux, so we have # to configure the index url for cpu-only pytorch manually From 3a954b63ca3837748e146ff1d1ebe395abd20717 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 18:00:37 +0000 Subject: [PATCH 022/225] Use setup-python --- .github/workflows/packaging_wheels.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c91be384..c54bcbca 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -66,7 +66,7 @@ jobs: steps: - name: Checkout DuckDB Python - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 @@ -100,7 +100,13 @@ jobs: - name: Set OVERRIDE_GIT_DESCRIBE if: ${{ inputs.set-version != '' }} run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV - + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ + fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] + }} + allow-prereleases: true # Install Astral UV, which will be used as build-frontend for cibuildwheel - uses: astral-sh/setup-uv@v6 with: From 9d66ab1249742c225ec8d439e5945d494559c661 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 18:29:53 +0000 Subject: [PATCH 023/225] Close quotes. --- .github/workflows/packaging_wheels.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c54bcbca..5788d5e0 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -58,10 +58,9 @@ jobs: ls -la pylock.toml 2>/dev/null || echo "pylock.toml not created" && echo "done" CIBW_TEST_COMMAND: > - uv run python -c "import duckdb + uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - CCACHE_DIR: > - /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + CCACHE_DIR: build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE: 2G steps: @@ -82,7 +81,7 @@ jobs: path: | build/ wheelhouse/ - /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- @@ -136,7 +135,10 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G + CIBW_BEFORE_ALL_LINUX: | + yum install -y ccache || apt-get update && apt-get install -y ccache + mkdir -p build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - name: Upload wheel uses: actions/upload-artifact@v4 with: @@ -150,5 +152,5 @@ jobs: path: | build/ wheelhouse/ - /tmp/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} \ No newline at end of file From d0822a4385c1bc9f77c516afb9af2155935fc973 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 18:33:36 +0000 Subject: [PATCH 024/225] testing ccache --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5788d5e0..1dc7ea30 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -62,7 +62,7 @@ jobs: #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs CCACHE_DIR: build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE: 2G - + steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 From 49974ac5690639de31f0c3da77613acd503e471f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 18:50:01 +0000 Subject: [PATCH 025/225] reenable ccache --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 1dc7ea30..b18d89b1 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -135,7 +135,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" CIBW_BEFORE_ALL_LINUX: | yum install -y ccache || apt-get update && apt-get install -y ccache mkdir -p build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} From 21235c4d6cb40114c6dfe45ecf7e49000a714da9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:01:08 +0000 Subject: [PATCH 026/225] cache lock free --- .github/workflows/packaging_wheels.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b18d89b1..89f33b3e 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -60,7 +60,7 @@ jobs: CIBW_TEST_COMMAND: > uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - CCACHE_DIR: build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + CCACHE_DIR: build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE: 2G steps: @@ -81,7 +81,7 @@ jobs: path: | build/ wheelhouse/ - build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- @@ -135,22 +135,27 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" CIBW_BEFORE_ALL_LINUX: | yum install -y ccache || apt-get update && apt-get install -y ccache - mkdir -p build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + mkdir -p build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} + echo "Initial ccache stats:" && ccache -s + CIBW_BEFORE_BUILD: | + echo "Pre-build ccache stats:" && ccache -s - name: Upload wheel uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl - compression-level: 0 + compression-level: 0 - name: Save Build Cache if: always() # Always save, even on failure - uses: actions/cache/save@v4 + uses: pat-s/always-upload-cache@v3.0.11 + with: + name: build-cache with: path: | build/ wheelhouse/ - build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} \ No newline at end of file From ab98aa1897f3e3ffceabf6bc9b967364f4eab6af Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:04:28 +0000 Subject: [PATCH 027/225] ccache --- .github/workflows/packaging_wheels.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 89f33b3e..6991f0e2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -139,20 +139,15 @@ jobs: CIBW_BEFORE_ALL_LINUX: | yum install -y ccache || apt-get update && apt-get install -y ccache mkdir -p build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} - echo "Initial ccache stats:" && ccache -s - CIBW_BEFORE_BUILD: | - echo "Pre-build ccache stats:" && ccache -s - name: Upload wheel uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl - compression-level: 0 + compression-level: 0 - name: Save Build Cache if: always() # Always save, even on failure - uses: pat-s/always-upload-cache@v3.0.11 - with: - name: build-cache + uses: actions/cache/save@v4 with: path: | build/ From d0331102c6ca853308dd21564abec294436da3e6 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:14:33 +0000 Subject: [PATCH 028/225] debugging project dir --- .github/workflows/packaging_wheels.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 6991f0e2..d9de55c2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -60,7 +60,7 @@ jobs: CIBW_TEST_COMMAND: > uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - CCACHE_DIR: build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} + CCACHE_DIR: ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE: 2G steps: @@ -81,7 +81,7 @@ jobs: path: | build/ wheelhouse/ - build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} + ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- @@ -135,10 +135,15 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" CIBW_BEFORE_ALL_LINUX: | yum install -y ccache || apt-get update && apt-get install -y ccache - mkdir -p build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} + pwd + ls -la + echo " ccache dir" + mkdir -p ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + echo "Test file in project root" > test-mount-check.txt + ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - name: Upload wheel uses: actions/upload-artifact@v4 with: @@ -152,5 +157,5 @@ jobs: path: | build/ wheelhouse/ - build/ccache-b-${{ matrix.python }}-${{ matrix.platform.arch }} + ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }} \ No newline at end of file From 8498e624cd9e78cf3edf45c7333827cbcbfd9893 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:20:58 +0000 Subject: [PATCH 029/225] add debug info --- .github/workflows/packaging_wheels.yml | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index d9de55c2..7d37c550 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -144,12 +144,43 @@ jobs: mkdir -p ccache-${{ matrix.python }}-${{ matrix.platform.arch }} echo "Test file in project root" > test-mount-check.txt ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + which gcc g++ ccache + echo "CC=$CC CXX=$CXX" + ccache -s + CIBW_BEFORE_BUILD: | + echo "Container PWD: $(pwd)" + echo "CCACHE_DIR env var: $CCACHE_DIR" + echo "Checking if CCACHE_DIR exists as absolute path:" + ls -la "$CCACHE_DIR" || echo "CCACHE_DIR path not found" + echo "Checking if ccache dir exists in current directory:" + ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not in current dir" + echo "Full filesystem check:" + find . -name "*ccache*" -type d 2>/dev/null || echo "no ccache dirs found anywhere" + ccache -s + CIBW_REPAIR_WHEEL_COMMAND_LINUX: | + ccache -s + echo "ccache directory contents:" + find ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | head -10 || echo "no cache files found" + du -sh ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" + auditwheel repair -w {dest_dir} {wheel} - name: Upload wheel uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl compression-level: 0 + - name: Check final ccache state + if: always() + run: | + echo "Host PWD: $(pwd)" + echo "Checking ccache directory on host:" + ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not found on host" + echo "Directory size:" + du -sh ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" + echo "Cache files count:" + find ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | wc -l || echo "0" + echo "Host ccache stats (if available):" + CCACHE_DIR=ccache-${{ matrix.python }}-${{ matrix.platform.arch }} ccache -s || echo "ccache not available on host" - name: Save Build Cache if: always() # Always save, even on failure uses: actions/cache/save@v4 From 17cd50f4912699bedba3db6a594fdf2bab2a8aff Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:26:52 +0000 Subject: [PATCH 030/225] make sure the cache dir is mounted --- .github/workflows/packaging_wheels.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 7d37c550..55f1c2df 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -87,6 +87,12 @@ jobs: build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Show ccache stats run: ccache -s + - name: Create ccache directory on host + run: | + mkdir -p ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + echo "Created ccache directory on host at $(date)" > ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt + echo "Created ccache directory on host" + ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -154,6 +160,8 @@ jobs: ls -la "$CCACHE_DIR" || echo "CCACHE_DIR path not found" echo "Checking if ccache dir exists in current directory:" ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not in current dir" + echo "Verifying host-created placeholder file:" + cat ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt 2>/dev/null || echo "placeholder file not found - mount issue!" echo "Full filesystem check:" find . -name "*ccache*" -type d 2>/dev/null || echo "no ccache dirs found anywhere" ccache -s From 60953356ee94aff119d608e20f1c9186c82b8c65 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:30:30 +0000 Subject: [PATCH 031/225] change cache key --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 55f1c2df..520a2c1c 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -82,7 +82,7 @@ jobs: build/ wheelhouse/ ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - key: build-dir-${{ matrix.python }}-${{ runner.os }} + key: build-dir-${{ matrix.python }}-${{ runner.os }}-b restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Show ccache stats @@ -197,4 +197,4 @@ jobs: build/ wheelhouse/ ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - key: build-dir-${{ matrix.python }}-${{ runner.os }} \ No newline at end of file + key: build-dir-${{ matrix.python }}-${{ runner.os }}-b \ No newline at end of file From 4e5cfdbc29292b70c4168a35b44384a34f78c763 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:36:56 +0000 Subject: [PATCH 032/225] try to force ccache --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 520a2c1c..c919702b 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -141,7 +141,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache CIBW_BEFORE_ALL_LINUX: | yum install -y ccache || apt-get update && apt-get install -y ccache pwd From 32caa8b3879a9d932db17a752110448d4a4d76f0 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:44:39 +0000 Subject: [PATCH 033/225] add test file, still figuring out ccache caching --- .github/workflows/packaging_wheels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c919702b..f5516b9b 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -170,6 +170,9 @@ jobs: echo "ccache directory contents:" find ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | head -10 || echo "no cache files found" du -sh ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" + echo "Creating test file from container..." + echo "Test file created at $(date)" > ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/test-from-container.txt + ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ auditwheel repair -w {dest_dir} {wheel} - name: Upload wheel uses: actions/upload-artifact@v4 From 04fdedd1051eb17b8d152f0169df09b2767b2959 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:44:56 +0000 Subject: [PATCH 034/225] test file --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index f5516b9b..6fd9faa1 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -170,7 +170,7 @@ jobs: echo "ccache directory contents:" find ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | head -10 || echo "no cache files found" du -sh ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" - echo "Creating test file from container..." + echo " test file from container" echo "Test file created at $(date)" > ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/test-from-container.txt ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ auditwheel repair -w {dest_dir} {wheel} From b2757fdad0f810e457540c748ff30c7565e6950a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:49:41 +0000 Subject: [PATCH 035/225] move into build dir --- .github/workflows/packaging_wheels.yml | 36 ++++++++++++-------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 6fd9faa1..140b2167 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -60,7 +60,7 @@ jobs: CIBW_TEST_COMMAND: > uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - CCACHE_DIR: ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + CCACHE_DIR: build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE: 2G steps: @@ -81,7 +81,6 @@ jobs: path: | build/ wheelhouse/ - ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }}-b restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- @@ -89,10 +88,10 @@ jobs: run: ccache -s - name: Create ccache directory on host run: | - mkdir -p ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - echo "Created ccache directory on host at $(date)" > ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt + mkdir -p build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + echo "Created ccache directory on host at $(date)" > build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt echo "Created ccache directory on host" - ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ + ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -141,15 +140,15 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache CIBW_BEFORE_ALL_LINUX: | yum install -y ccache || apt-get update && apt-get install -y ccache pwd ls -la echo " ccache dir" - mkdir -p ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + mkdir -p build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} echo "Test file in project root" > test-mount-check.txt - ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }} + ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} which gcc g++ ccache echo "CC=$CC CXX=$CXX" ccache -s @@ -159,20 +158,20 @@ jobs: echo "Checking if CCACHE_DIR exists as absolute path:" ls -la "$CCACHE_DIR" || echo "CCACHE_DIR path not found" echo "Checking if ccache dir exists in current directory:" - ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not in current dir" + ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not in current dir" echo "Verifying host-created placeholder file:" - cat ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt 2>/dev/null || echo "placeholder file not found - mount issue!" + cat build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt 2>/dev/null || echo "placeholder file not found - mount issue!" echo "Full filesystem check:" find . -name "*ccache*" -type d 2>/dev/null || echo "no ccache dirs found anywhere" ccache -s CIBW_REPAIR_WHEEL_COMMAND_LINUX: | ccache -s echo "ccache directory contents:" - find ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | head -10 || echo "no cache files found" - du -sh ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" + find build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | head -10 || echo "no cache files found" + du -sh build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" echo " test file from container" - echo "Test file created at $(date)" > ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/test-from-container.txt - ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ + echo "Test file created at $(date)" > build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/test-from-container.txt + ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ auditwheel repair -w {dest_dir} {wheel} - name: Upload wheel uses: actions/upload-artifact@v4 @@ -185,13 +184,13 @@ jobs: run: | echo "Host PWD: $(pwd)" echo "Checking ccache directory on host:" - ls -la ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not found on host" + ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not found on host" echo "Directory size:" - du -sh ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" + du -sh build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" echo "Cache files count:" - find ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | wc -l || echo "0" + find build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | wc -l || echo "0" echo "Host ccache stats (if available):" - CCACHE_DIR=ccache-${{ matrix.python }}-${{ matrix.platform.arch }} ccache -s || echo "ccache not available on host" + CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} ccache -s || echo "ccache not available on host" - name: Save Build Cache if: always() # Always save, even on failure uses: actions/cache/save@v4 @@ -199,5 +198,4 @@ jobs: path: | build/ wheelhouse/ - ccache-${{ matrix.python }}-${{ matrix.platform.arch }} key: build-dir-${{ matrix.python }}-${{ runner.os }}-b \ No newline at end of file From d259d1931eef38a35b9d8f3dd6635b03cb37684d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 19:54:11 +0000 Subject: [PATCH 036/225] simnplify --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 140b2167..0d32d490 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -140,7 +140,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" CIBW_BEFORE_ALL_LINUX: | yum install -y ccache || apt-get update && apt-get install -y ccache pwd From fae60ce6a8c2d1885c3d08a1931c46b2718555c7 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 20:10:14 +0000 Subject: [PATCH 037/225] option 2: sccache --- .github/workflows/packaging_wheels.yml | 69 ++++++-------------------- 1 file changed, 16 insertions(+), 53 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 0d32d490..76251179 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -60,8 +60,6 @@ jobs: CIBW_TEST_COMMAND: > uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - CCACHE_DIR: build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - CCACHE_MAXSIZE: 2G steps: - name: Checkout DuckDB Python @@ -70,11 +68,8 @@ jobs: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - - name: Install ccache - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: ccache - version: 1.0 + - name: Setup sccache # idea from https://github.com/pypa/cibuildwheel/issues/1030 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Restore Build Cache uses: actions/cache/restore@v4 with: @@ -84,14 +79,6 @@ jobs: key: build-dir-${{ matrix.python }}-${{ runner.os }}-b restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- - - name: Show ccache stats - run: ccache -s - - name: Create ccache directory on host - run: | - mkdir -p build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - echo "Created ccache directory on host at $(date)" > build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt - echo "Created ccache directory on host" - ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -140,38 +127,21 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_MAXSIZE=2G CC="ccache gcc" CXX="ccache g++" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true CC="sccache gcc" CXX="sccache g++" CIBW_BEFORE_ALL_LINUX: | - yum install -y ccache || apt-get update && apt-get install -y ccache - pwd - ls -la - echo " ccache dir" - mkdir -p build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - echo "Test file in project root" > test-mount-check.txt - ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} - which gcc g++ ccache + curl -L https://github.com/mozilla/sccache/releases/download/v0.7.7/sccache-v0.7.7-x86_64-unknown-linux-musl.tar.gz | tar xz + chmod +x sccache-v0.7.7-x86_64-unknown-linux-musl/sccache + mv sccache-v0.7.7-x86_64-unknown-linux-musl/sccache /usr/local/bin/ + echo "Verifying sccache installation" + which gcc g++ sccache echo "CC=$CC CXX=$CXX" - ccache -s + sccache --show-stats CIBW_BEFORE_BUILD: | - echo "Container PWD: $(pwd)" - echo "CCACHE_DIR env var: $CCACHE_DIR" - echo "Checking if CCACHE_DIR exists as absolute path:" - ls -la "$CCACHE_DIR" || echo "CCACHE_DIR path not found" - echo "Checking if ccache dir exists in current directory:" - ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not in current dir" - echo "Verifying host-created placeholder file:" - cat build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/host-created.txt 2>/dev/null || echo "placeholder file not found - mount issue!" - echo "Full filesystem check:" - find . -name "*ccache*" -type d 2>/dev/null || echo "no ccache dirs found anywhere" - ccache -s + echo "sccache status before build:" + sccache --show-stats CIBW_REPAIR_WHEEL_COMMAND_LINUX: | - ccache -s - echo "ccache directory contents:" - find build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | head -10 || echo "no cache files found" - du -sh build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" - echo " test file from container" - echo "Test file created at $(date)" > build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/test-from-container.txt - ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ + echo "Final sccache stats:" + sccache --show-stats auditwheel repair -w {dest_dir} {wheel} - name: Upload wheel uses: actions/upload-artifact@v4 @@ -179,18 +149,11 @@ jobs: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl compression-level: 0 - - name: Check final ccache state + - name: Show final sccache stats if: always() run: | - echo "Host PWD: $(pwd)" - echo "Checking ccache directory on host:" - ls -la build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir not found on host" - echo "Directory size:" - du -sh build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ || echo "ccache dir missing" - echo "Cache files count:" - find build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }}/ -type f | wc -l || echo "0" - echo "Host ccache stats (if available):" - CCACHE_DIR=build/ccache-${{ matrix.python }}-${{ matrix.platform.arch }} ccache -s || echo "ccache not available on host" + echo "Final sccache statistics on host:" + sccache --show-stats || echo "sccache not available on host" - name: Save Build Cache if: always() # Always save, even on failure uses: actions/cache/save@v4 From b144404776913110666409ea33855e263950eddc Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 20:16:33 +0000 Subject: [PATCH 038/225] sccache --- .github/workflows/packaging_wheels.yml | 14 +++----------- pyproject.toml | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 76251179..85daf394 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -76,7 +76,7 @@ jobs: path: | build/ wheelhouse/ - key: build-dir-${{ matrix.python }}-${{ runner.os }}-b + key: build-dir-${{ matrix.python }}-${{ runner.os }}-c restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Checkout DuckDB @@ -127,15 +127,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true CC="sccache gcc" CXX="sccache g++" - CIBW_BEFORE_ALL_LINUX: | - curl -L https://github.com/mozilla/sccache/releases/download/v0.7.7/sccache-v0.7.7-x86_64-unknown-linux-musl.tar.gz | tar xz - chmod +x sccache-v0.7.7-x86_64-unknown-linux-musl/sccache - mv sccache-v0.7.7-x86_64-unknown-linux-musl/sccache /usr/local/bin/ - echo "Verifying sccache installation" - which gcc g++ sccache - echo "CC=$CC CXX=$CXX" - sccache --show-stats + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CC="sccache gcc" CXX="sccache g++" CIBW_BEFORE_BUILD: | echo "sccache status before build:" sccache --show-stats @@ -161,4 +153,4 @@ jobs: path: | build/ wheelhouse/ - key: build-dir-${{ matrix.python }}-${{ runner.os }}-b \ No newline at end of file + key: build-dir-${{ matrix.python }}-${{ runner.os }}-c \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7b29c6fe..f766f64c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,10 +383,19 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -before-build = ["yum install -y ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN"] +before-build = [ + "curl -L https://github.com/mozilla/sccache/releases/download/v0.7.7/sccache-v0.7.7-x86_64-unknown-linux-musl.tar.gz | tar xz", + "chmod +x sccache-v0.7.7-x86_64-unknown-linux-musl/sccache", + "mv sccache-v0.7.7-x86_64-unknown-linux-musl/sccache /usr/local/bin/", + "which gcc g++ sccache", + "sccache --show-stats" +] [tool.cibuildwheel.macos] -before-build = ["brew install ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN"] +before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -before-build = ["choco install ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN"] +before-build = ["choco install sccache"] From 15361cf315d9d3b9463ffbab4bc80611a16b319f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 20:48:33 +0000 Subject: [PATCH 039/225] try --- .github/workflows/packaging_wheels.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 85daf394..a538b0b6 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -128,9 +128,6 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CC="sccache gcc" CXX="sccache g++" - CIBW_BEFORE_BUILD: | - echo "sccache status before build:" - sccache --show-stats CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" sccache --show-stats From 89ad08c0e8b7f69f2fcffbf7e4c5ffd5306484dc Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 20:52:25 +0000 Subject: [PATCH 040/225] increase build --- .github/workflows/packaging_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a538b0b6..a2619a16 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -128,6 +128,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CC="sccache gcc" CXX="sccache g++" + CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" sccache --show-stats From e7b438ab766389aff660fdd5bdd0d7897002b8a6 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 20:53:48 +0000 Subject: [PATCH 041/225] pass env --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a2619a16..b7352c19 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -127,7 +127,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CC="sccache gcc" CXX="sccache g++" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} ACTIONS_CACHE_URL=$ACTIONS_CACHE_URL ACTIONS_RUNTIME_TOKEN=$ACTIONS_RUNTIME_TOKEN CC="sccache gcc" CXX="sccache g++" CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 27b42520ac11d1a2faae3cfa24e30012f98d11f6 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:01:58 +0000 Subject: [PATCH 042/225] use v.0.10.0 sccache --- .github/workflows/packaging_wheels.yml | 4 +++- pyproject.toml | 15 +++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b7352c19..628f5ede 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -70,6 +70,8 @@ jobs: submodules: true - name: Setup sccache # idea from https://github.com/pypa/cibuildwheel/issues/1030 uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" - name: Restore Build Cache uses: actions/cache/restore@v4 with: @@ -127,7 +129,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} ACTIONS_CACHE_URL=$ACTIONS_CACHE_URL ACTIONS_RUNTIME_TOKEN=$ACTIONS_RUNTIME_TOKEN CC="sccache gcc" CXX="sccache g++" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER=sccache CMAKE_CXX_COMPILER_LAUNCHER=sccache CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" diff --git a/pyproject.toml b/pyproject.toml index f766f64c..ed32a8bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,19 +383,18 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = [ - "curl -L https://github.com/mozilla/sccache/releases/download/v0.7.7/sccache-v0.7.7-x86_64-unknown-linux-musl.tar.gz | tar xz", - "chmod +x sccache-v0.7.7-x86_64-unknown-linux-musl/sccache", - "mv sccache-v0.7.7-x86_64-unknown-linux-musl/sccache /usr/local/bin/", - "which gcc g++ sccache", - "sccache --show-stats" + "wget https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz", + "tar -zxf sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz", + "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin" ] +before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = ["choco install sccache"] From d7884795e616191cc1afa3355cea2b2bfdbc2cb2 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:04:49 +0000 Subject: [PATCH 043/225] curl not wget --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed32a8bf..99bec857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -385,9 +385,9 @@ enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = [ - "wget https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz", - "tar -zxf sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz", - "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin" + "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", + "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", + "which sccache" ] before-test = "sccache --show-stats" From 8a9ee03843b419e244eeca525edcd9da88ad45dd Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:09:36 +0000 Subject: [PATCH 044/225] double sccache --- .github/workflows/packaging_wheels.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 628f5ede..b82acc18 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -78,7 +78,8 @@ jobs: path: | build/ wheelhouse/ - key: build-dir-${{ matrix.python }}-${{ runner.os }}-c + /home/runner/.cache/cibuildwheel + key: build-dir-${{ matrix.python }}-${{ runner.os }}-d restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Checkout DuckDB @@ -129,7 +130,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER=sccache CMAKE_CXX_COMPILER_LAUNCHER=sccache + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CC="sccache gcc" CXX="sccache g++" CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" @@ -153,4 +154,5 @@ jobs: path: | build/ wheelhouse/ - key: build-dir-${{ matrix.python }}-${{ runner.os }}-c \ No newline at end of file + /home/runner/.cache/cibuildwheel + key: build-dir-${{ matrix.python }}-${{ runner.os }}-d \ No newline at end of file From d7de926de6029c14d4124ec9f9409e1a85d11528 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:12:53 +0000 Subject: [PATCH 045/225] triple? Wrong direction. --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b82acc18..bb1aed80 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -130,7 +130,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CC="sccache gcc" CXX="sccache g++" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER=sccache CMAKE_CXX_COMPILER_LAUNCHER=sccache CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 11b014446e55dd1d765a22786c931482da87bc6d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:16:48 +0000 Subject: [PATCH 046/225] double the fun --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index bb1aed80..022a1856 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -130,7 +130,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER=sccache CMAKE_CXX_COMPILER_LAUNCHER=sccache + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" diff --git a/pyproject.toml b/pyproject.toml index 99bec857..edfc0ba4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,7 +383,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", @@ -392,9 +392,9 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER"] before-build = ["choco install sccache"] From eed25a66994c75ee1e5afd3034aaef74350e0502 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:21:30 +0000 Subject: [PATCH 047/225] duckdb already sets sccache --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index edfc0ba4..99bec857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,7 +383,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", @@ -392,9 +392,9 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = ["choco install sccache"] From 2bb92787a1d3d8830e15e42f24a606c1c5d20a8f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:35:25 +0000 Subject: [PATCH 048/225] double => 1 --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 022a1856..35975853 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -130,7 +130,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 8a761a93bc07636da47573237978f9b2aca7eb25 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 21:57:08 +0000 Subject: [PATCH 049/225] test again --- .github/workflows/packaging_wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 35975853..5d492c27 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -79,7 +79,7 @@ jobs: build/ wheelhouse/ /home/runner/.cache/cibuildwheel - key: build-dir-${{ matrix.python }}-${{ runner.os }}-d + key: build-dir-${{ matrix.python }}-${{ runner.os }}-e restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Checkout DuckDB @@ -105,7 +105,7 @@ jobs: - uses: astral-sh/setup-uv@v6 with: version: "0.8.16" - enable-cache: true + enable-cache: false cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} python-version: ${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] @@ -155,4 +155,4 @@ jobs: build/ wheelhouse/ /home/runner/.cache/cibuildwheel - key: build-dir-${{ matrix.python }}-${{ runner.os }}-d \ No newline at end of file + key: build-dir-${{ matrix.python }}-${{ runner.os }}-e \ No newline at end of file From 8efe5630aaa14b1ef6efbf391a982642b9dc3543 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 22:09:29 +0000 Subject: [PATCH 050/225] Pass variables explicitly --- .github/workflows/packaging_wheels.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5d492c27..6269df09 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -50,13 +50,8 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - pwd && ls -la && - python --version && uv --version && - echo "uv export" && - uv export --only-group test --no-emit-project --output-file pylock.toml --directory /project --verbose 2>&1 || echo "Export failed with exit code $?" && - echo "listing file" && - ls -la pylock.toml 2>/dev/null || echo "pylock.toml not created" && - echo "done" + uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && + uv pip install -r pylock.toml CIBW_TEST_COMMAND: > uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs @@ -66,7 +61,7 @@ jobs: uses: actions/checkout@v5 with: ref: ${{ inputs.duckdb-python-sha }} - fetch-depth: 0 + fetch-depth: 0 submodules: true - name: Setup sccache # idea from https://github.com/pypa/cibuildwheel/issues/1030 uses: mozilla-actions/sccache-action@v0.0.9 @@ -130,7 +125,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} ACTIONS_CACHE_URL=$ACTIONS_CACHE_URL ACTIONS_RUNTIME_TOKEN=$ACTIONS_RUNTIME_TOKEN CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From f3c9431c205190746654aa3e5a041546bba57d6a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 22:20:51 +0000 Subject: [PATCH 051/225] pass env --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 6269df09..e1da3186 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -125,7 +125,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} ACTIONS_CACHE_URL=$ACTIONS_CACHE_URL ACTIONS_RUNTIME_TOKEN=$ACTIONS_RUNTIME_TOKEN + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 2b65e520094794ae22b53ad0f868f9ae05b5ed3a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 23:01:40 +0000 Subject: [PATCH 052/225] export --- .github/workflows/packaging_wheels.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index e1da3186..c53c74cd 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -67,6 +67,12 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.9 with: version: "v0.10.0" + - name: Configure sccache + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Restore Build Cache uses: actions/cache/restore@v4 with: From 2853060e06d0a317c3148f3bf12869b6b9f70b05 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 23:28:16 +0000 Subject: [PATCH 053/225] verify vars --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c53c74cd..120c2e78 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -80,7 +80,7 @@ jobs: build/ wheelhouse/ /home/runner/.cache/cibuildwheel - key: build-dir-${{ matrix.python }}-${{ runner.os }}-e + key: build-dir-${{ matrix.python }}-${{ runner.os }}- restore-keys: | build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Checkout DuckDB diff --git a/pyproject.toml b/pyproject.toml index 99bec857..c7d84e8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -387,7 +387,10 @@ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", - "which sccache" + "which sccache", + "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", + "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", + "echo \"GitHub Actions environment variables verified in container\"" ] before-test = "sccache --show-stats" From 36dd8a27ea325792ad19ba647ce6aa8417aa2cb0 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Wed, 10 Sep 2025 23:59:51 +0000 Subject: [PATCH 054/225] fix conditional logic for 3.14 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c7d84e8a..a4ebe155 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,7 +244,7 @@ test = [ # dependencies used for running tests "fsspec>=2022.11.0", "pandas>=2.0.0; python_version < '3.14'", "pyarrow>=18.0.0; python_version < '3.14'", - "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'", + "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'win32' and python_version < '3.12'", From 4df812db3842495f3072764834de4b4460e8c423 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 00:04:12 +0000 Subject: [PATCH 055/225] debug logging --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 120c2e78..6c3f460d 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -131,7 +131,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 28556cd8064eff97da1591153ce33c2057435246 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 00:13:37 +0000 Subject: [PATCH 056/225] true not on --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 6c3f460d..a83fd498 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -131,7 +131,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 6ed58ec3629535bef115bf3697ac8cc451759680 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 00:43:58 +0000 Subject: [PATCH 057/225] v2 --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a4ebe155..c7417023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,7 +383,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", @@ -395,9 +395,9 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_CACHE_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] before-build = ["choco install sccache"] From 4cad1245543840294336f445e9ae056fe82c78da Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 01:07:57 +0000 Subject: [PATCH 058/225] self hosted, test one file --- .github/workflows/packaging_wheels.yml | 32 ++++++++++++++++++++++++++ pyproject.toml | 8 ++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a83fd498..1fed1c54 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -29,6 +29,7 @@ jobs: name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}' strategy: fail-fast: false + runs-on: self-hosted matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: @@ -67,6 +68,37 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.9 with: version: "v0.10.0" + + - name: Install System Dependencies (APT - Ubuntu/Debian) + if: steps.detect-pm.outputs.package_manager == 'apt' + run: | + echo "Installing build dependencies with APT" + sudo apt update + sudo apt-get install -y \ + build-essential \ + cmake \ + git \ + wget \ + curl \ + pkg-config \ + libbz2-dev \ + libffi-dev \ + libgdbm-dev \ + libgdbm-compat-dev \ + liblzma-dev \ + libncurses5-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + lzma \ + lzma-dev \ + tk-dev \ + uuid-dev \ + zlib1g-dev \ + clang \ + llvm + echo "APT package installation completed" + - name: Configure sccache uses: actions/github-script@v7 with: diff --git a/pyproject.toml b/pyproject.toml index c7417023..08456f8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -390,7 +390,13 @@ before-build = [ "which sccache", "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", - "echo \"GitHub Actions environment variables verified in container\"" + "echo \"GitHub Actions environment variables verified in container\"", + "echo \"Testing sccache GitHub Actions cache access...\"", + "SCCACHE_LOG=debug sccache --start-server 2>&1 | head -20 || true", + "echo 'int main(){return 0;}' > test.c", + "SCCACHE_LOG=debug sccache gcc -c test.c -o test.o 2>&1 | head -20 || true", + "sccache --show-stats", + "echo \"Sccache test complete\"" ] before-test = "sccache --show-stats" From aa7a9741f50fa30d8b142a9bfc6639f63a3742b0 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 01:09:17 +0000 Subject: [PATCH 059/225] self hosted --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 1fed1c54..db5c0a53 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -27,9 +27,9 @@ on: jobs: build_wheels: name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}' + runs-on: self-hosted strategy: fail-fast: false - runs-on: self-hosted matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: From aba20d8a958970321b2a69eee3d158c58cdb94e0 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 01:10:27 +0000 Subject: [PATCH 060/225] self hosted 2 --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index db5c0a53..be48ab27 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -27,14 +27,14 @@ on: jobs: build_wheels: name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}' - runs-on: self-hosted strategy: fail-fast: false matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: + - { os: self-hosted, arch: x86_64, cibw_system: manylinux} #- { os: windows-2025, arch: amd64, cibw_system: win } - - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } From f705a883d75b005104410d6463cfc9d9ba39c196 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 01:13:53 +0000 Subject: [PATCH 061/225] fix python version typo --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index be48ab27..f62268ab 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -127,7 +127,7 @@ jobs: - name: Set OVERRIDE_GIT_DESCRIBE if: ${{ inputs.set-version != '' }} run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v6 with: python-version: ${{ From 97a0f50a9718d60d44e1bb589d955ad0feaec2a5 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 01:27:20 +0000 Subject: [PATCH 062/225] start the server --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 08456f8e..c1b9120d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -391,8 +391,9 @@ before-build = [ "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", "echo \"GitHub Actions environment variables verified in container\"", - "echo \"Testing sccache GitHub Actions cache access...\"", - "SCCACHE_LOG=debug sccache --start-server 2>&1 | head -20 || true", + "echo \"Testing sccache GitHub Actions cache access...\"", + "sccache --start-server", + "SCCACHE_LOG=debug sccache --show-stats 2>&1 | head -10 || true", "echo 'int main(){return 0;}' > test.c", "SCCACHE_LOG=debug sccache gcc -c test.c -o test.o 2>&1 | head -20 || true", "sccache --show-stats", From 777c9d892d18325395aeba151458fb92725cef0b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 01:42:57 +0000 Subject: [PATCH 063/225] a --- .github/workflows/packaging_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index f62268ab..9d67eba9 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -161,6 +161,7 @@ jobs: - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels uses: pypa/cibuildwheel@v3.0 env: + CIBW_TOOL_PYTHON: /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug From 75566321dad4a5ec3f79954f6572c9d70bda9b30 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 02:40:15 +0000 Subject: [PATCH 064/225] add debug --- .github/workflows/packaging_wheels.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9d67eba9..c277329b 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -158,6 +158,14 @@ jobs: exit 1 fi + - name: Debug Python paths + run: | + echo "Current python: $(which python)" + echo "Python version: $(python --version)" + echo "pythonLocation: $pythonLocation" + echo "Checking CIBW_TOOL_PYTHON path: /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python" + ls -la /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python || echo "Path not found" + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels uses: pypa/cibuildwheel@v3.0 env: From 753e59a2f238157e43dd01c53e2086676a6fc1cf Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 02:42:26 +0000 Subject: [PATCH 065/225] bump --- .github/workflows/packaging_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c277329b..a45b32d2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -157,6 +157,7 @@ jobs: echo "Python version mismatch!" exit 1 fi + - name: Debug Python paths run: | From f2c5a7526572528a3dd7378e9a6cbf80e13af748 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:01:53 +0000 Subject: [PATCH 066/225] add debug --- .github/workflows/packaging_wheels.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a45b32d2..ddddb1bb 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -157,7 +157,7 @@ jobs: echo "Python version mismatch!" exit 1 fi - + - name: Debug Python paths run: | @@ -167,10 +167,16 @@ jobs: echo "Checking CIBW_TOOL_PYTHON path: /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python" ls -la /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python || echo "Path not found" + - name: Debug CIBW_TOOL_PYTHON path + run: | + echo "pythonLocation: $pythonLocation" + echo "Expected CIBW_TOOL_PYTHON: $pythonLocation/bin/python" + ls -la "$pythonLocation/bin/python" || echo "Python not found at expected location" + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels uses: pypa/cibuildwheel@v3.0 env: - CIBW_TOOL_PYTHON: /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python + CIBW_TOOL_PYTHON: ${{ env.pythonLocation }}/bin/python CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug From c50dc8f2025032e64d08b740f57d71c5830e700c Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:16:47 +0000 Subject: [PATCH 067/225] install directly --- .github/workflows/packaging_wheels.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index ddddb1bb..b17ce88e 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -128,7 +128,7 @@ jobs: if: ${{ inputs.set-version != '' }} run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: ${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] @@ -157,7 +157,7 @@ jobs: echo "Python version mismatch!" exit 1 fi - +W - name: Debug Python paths run: | @@ -173,10 +173,12 @@ jobs: echo "Expected CIBW_TOOL_PYTHON: $pythonLocation/bin/python" ls -la "$pythonLocation/bin/python" || echo "Python not found at expected location" + - name: Install cibuildwheel + run: python -m pip install cibuildwheel + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - uses: pypa/cibuildwheel@v3.0 + run: python -m cibuildwheel --output-dir wheelhouse env: - CIBW_TOOL_PYTHON: ${{ env.pythonLocation }}/bin/python CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug From e8c27b8ab30e7629ca4b5f7f2823b1893f892a60 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:17:49 +0000 Subject: [PATCH 068/225] typo --- .github/workflows/packaging_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b17ce88e..7a01e8f2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -157,7 +157,6 @@ jobs: echo "Python version mismatch!" exit 1 fi -W - name: Debug Python paths run: | From 4ded61b7b65d712c5194ca6e478e964ad17a1cef Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:35:48 +0000 Subject: [PATCH 069/225] test sccache --- .github/workflows/packaging_wheels.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 7a01e8f2..d228b12d 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -105,6 +105,26 @@ jobs: script: | core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + + - name: Test sccache on host + run: | + echo "Testing sccache GitHub Actions cache access on host..." + sccache --start-server + echo "Host environment variables:" + echo "ACTIONS_RESULTS_URL: ${ACTIONS_RESULTS_URL}" + echo "ACTIONS_RUNTIME_TOKEN: ${ACTIONS_RUNTIME_TOKEN:+SET}" + echo "SCCACHE_GHA_ENABLED: ${SCCACHE_GHA_ENABLED}" + echo "Initial sccache stats:" + sccache --show-stats + echo 'int main(){return 0;}' > test_host.c + echo "Compiling test file with sccache on host..." + SCCACHE_LOG=debug sccache gcc -c test_host.c -o test_host.o 2>&1 | head -20 || true + echo "Final sccache stats on host:" + sccache --show-stats + rm -f test_host.c test_host.o + echo "Host sccache test complete" + env: + SCCACHE_GHA_ENABLED: true - name: Restore Build Cache uses: actions/cache/restore@v4 with: From b0a9b10d858d108148cc01283f5a49b4a6927155 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:44:39 +0000 Subject: [PATCH 070/225] fix test --- .github/workflows/packaging_wheels.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index d228b12d..34aa41bf 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: CIBW_TEST_COMMAND: > uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - + SCCACHE_GHA_ENABLED: "true" steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 @@ -69,6 +69,13 @@ jobs: with: version: "v0.10.0" + - name: Configure sccache + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + - name: Install System Dependencies (APT - Ubuntu/Debian) if: steps.detect-pm.outputs.package_manager == 'apt' run: | @@ -99,13 +106,6 @@ jobs: llvm echo "APT package installation completed" - - name: Configure sccache - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - name: Test sccache on host run: | echo "Testing sccache GitHub Actions cache access on host..." From eb4fd064f694f090eb9396804f353c4acfb4cc89 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:51:41 +0000 Subject: [PATCH 071/225] grant write perm for sccache --- .github/workflows/packaging_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 34aa41bf..7bd10182 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -73,6 +73,7 @@ jobs: uses: actions/github-script@v7 with: script: | + core.exportVariable('SCCACHE_GHA_ENABLED', 'true'); core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); From ea0f775cca9ab7ab6f848932beeae19da8ae9c64 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:54:52 +0000 Subject: [PATCH 072/225] Add write perm --- .github/workflows/packaging_wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 7bd10182..9bef04da 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -27,6 +27,8 @@ on: jobs: build_wheels: name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}' + permissions: + actions: write # For GitHub Actions cache write access strategy: fail-fast: false matrix: From 37eaf01502c8930e0c4d162a4d8f89078f95c467 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 03:56:03 +0000 Subject: [PATCH 073/225] change concurrency --- .github/workflows/on_pr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 7a4669cb..7e4b5107 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -14,9 +14,9 @@ on: - '!.github/workflows/on_push.yml' - '!.github/workflows/coverage.yml' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true jobs: submodule_sanity_guard: From 6772ea52078768927ac8867d976483429fbf8314 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 04:09:21 +0000 Subject: [PATCH 074/225] back to action --- .github/workflows/packaging_wheels.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9bef04da..a8cb7aec 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -71,14 +71,6 @@ jobs: with: version: "v0.10.0" - - name: Configure sccache - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('SCCACHE_GHA_ENABLED', 'true'); - core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - name: Install System Dependencies (APT - Ubuntu/Debian) if: steps.detect-pm.outputs.package_manager == 'apt' run: | @@ -195,11 +187,10 @@ jobs: echo "Expected CIBW_TOOL_PYTHON: $pythonLocation/bin/python" ls -la "$pythonLocation/bin/python" || echo "Python not found at expected location" - - name: Install cibuildwheel - run: python -m pip install cibuildwheel - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - run: python -m cibuildwheel --output-dir wheelhouse + uses: pypa/cibuildwheel@v3.1.3 + with: + output-dir: wheelhouse env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} From 8337cc90f12fccb3764b5264682ce617d1a04d48 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 04:14:38 +0000 Subject: [PATCH 075/225] sccache --- .github/workflows/packaging_wheels.yml | 7 ++++--- pyproject.toml | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a8cb7aec..f146ab92 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -187,10 +187,11 @@ jobs: echo "Expected CIBW_TOOL_PYTHON: $pythonLocation/bin/python" ls -la "$pythonLocation/bin/python" || echo "Python not found at expected location" + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.21.3 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - uses: pypa/cibuildwheel@v3.1.3 - with: - output-dir: wheelhouse + run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} diff --git a/pyproject.toml b/pyproject.toml index c1b9120d..f3796c95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,12 +383,12 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_CACHE_URL"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", "which sccache", - "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", + "if [ -z \"$ACTIONS_CACHE_URL\" ]; then echo \"ERROR: ACTIONS_CACHE_URL not set in container\" && exit 1; fi", "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", "echo \"GitHub Actions environment variables verified in container\"", "echo \"Testing sccache GitHub Actions cache access...\"", @@ -402,9 +402,9 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_CACHE_URL"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_CACHE_URL"] before-build = ["choco install sccache"] From c71846832941e18c723c51d7add95e6fed8d7835 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 04:17:29 +0000 Subject: [PATCH 076/225] bump --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index f146ab92..5f144b6f 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -188,7 +188,7 @@ jobs: ls -la "$pythonLocation/bin/python" || echo "Python not found at expected location" - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.21.3 + run: python -m pip install cibuildwheel - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels run: python -m cibuildwheel --output-dir wheelhouse From 4a17f550408075baaedb105b42ae5cf4de4e7fb4 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 04:20:14 +0000 Subject: [PATCH 077/225] bump2 --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5f144b6f..de2e4ccb 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -188,7 +188,7 @@ jobs: ls -la "$pythonLocation/bin/python" || echo "Python not found at expected location" - name: Install cibuildwheel - run: python -m pip install cibuildwheel + run: python -m pip install cibuildwheel==3.1.4 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels run: python -m cibuildwheel --output-dir wheelhouse From f86ec1f09fa05a28a93d8aca65bf812ac8ea7a30 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 04:23:10 +0000 Subject: [PATCH 078/225] "ACTIONS_RESULTS_URL", --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f3796c95..7045dc12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,7 +383,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_CACHE_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", @@ -402,9 +402,9 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_CACHE_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_CACHE_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL"] before-build = ["choco install sccache"] From 9aed4ad3ef1b0a25fe9da88548f5bdc59214c22b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 04:27:24 +0000 Subject: [PATCH 079/225] set all --- .github/workflows/packaging_wheels.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index de2e4ccb..08ae3319 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -70,7 +70,13 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.9 with: version: "v0.10.0" - + - name: Configure sccache + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Install System Dependencies (APT - Ubuntu/Debian) if: steps.detect-pm.outputs.package_manager == 'apt' run: | From a37c59740abadf6602a882f116ccbbea56477338 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 11:18:28 +0000 Subject: [PATCH 080/225] upd --- .github/workflows/packaging_wheels.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 08ae3319..a150f080 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -58,7 +58,7 @@ jobs: CIBW_TEST_COMMAND: > uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_ENABLED: "on" steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 @@ -70,12 +70,11 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.9 with: version: "v0.10.0" - - name: Configure sccache + - name: Configure Cache Env uses: actions/github-script@v7 with: script: | core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Install System Dependencies (APT - Ubuntu/Debian) if: steps.detect-pm.outputs.package_manager == 'apt' @@ -106,7 +105,7 @@ jobs: clang \ llvm echo "APT package installation completed" - + - name: Test sccache on host run: | echo "Testing sccache GitHub Actions cache access on host..." @@ -125,7 +124,7 @@ jobs: rm -f test_host.c test_host.o echo "Host sccache test complete" env: - SCCACHE_GHA_ENABLED: true + SCCACHE_GHA_ENABLED: "on" - name: Restore Build Cache uses: actions/cache/restore@v4 with: @@ -201,7 +200,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=true SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From af6fa957a31ba6559740162d9ccc334db5670d17 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 11:22:45 +0000 Subject: [PATCH 081/225] ACTIONS_RESULTS_URL --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7045dc12..77b78943 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -388,7 +388,7 @@ before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", "which sccache", - "if [ -z \"$ACTIONS_CACHE_URL\" ]; then echo \"ERROR: ACTIONS_CACHE_URL not set in container\" && exit 1; fi", + "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", "echo \"GitHub Actions environment variables verified in container\"", "echo \"Testing sccache GitHub Actions cache access...\"", From 0edec7cfe58ceac0c3123da6ee2ee49bfc3cc618 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 11:37:13 +0000 Subject: [PATCH 082/225] try a writeable dir --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a150f080..39734bf8 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -200,7 +200,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" diff --git a/pyproject.toml b/pyproject.toml index 77b78943..25eda251 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -388,6 +388,8 @@ before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", "which sccache", + "mkdir -p /tmp/sccache-cache", + "export SCCACHE_DIR=/tmp/sccache-cache", "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", "echo \"GitHub Actions environment variables verified in container\"", From 20f514671dfd3b8dcd1fa91bc54dd2678504d396 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 11:39:47 +0000 Subject: [PATCH 083/225] pass dir through --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 25eda251..fcbcf7a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,7 +383,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL", "SCCACHE_DIR"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", From fea57f90afe1218c9c2e55ffbbfebde1fc8994ab Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 11:46:25 +0000 Subject: [PATCH 084/225] capture logging --- .github/workflows/packaging_wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 39734bf8..d9f954ab 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -59,6 +59,7 @@ jobs: uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" + SCCACHE_DIR: /tmp/sccache-cache steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 @@ -200,7 +201,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_START_SERVER=1 SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From ded10ed4606571af7b243dbefe2ba3284e070f22 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 11:49:37 +0000 Subject: [PATCH 085/225] SCCACHE_START_SERVER=1 --- .github/workflows/packaging_wheels.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index d9f954ab..8722e2fe 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -201,13 +201,11 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_START_SERVER=1 SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" - sccache --show-stats - auditwheel repair -w {dest_dir} {wheel} - - name: Upload wheel + sccache --show-statsKl uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} From 3d11b76ffc4281a24a4c4d94aec324ea803a19f9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 11:52:46 +0000 Subject: [PATCH 086/225] 2 --- .github/workflows/packaging_wheels.yml | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 8722e2fe..a49ce158 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -205,7 +205,9 @@ jobs: CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" - sccache --show-statsKl + sccache --show-stats + auditwheel repair -w {dest_dir} {wheel} + - name: Upload wheel uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} diff --git a/pyproject.toml b/pyproject.toml index fcbcf7a5..d0c0eaf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -388,6 +388,7 @@ before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", "which sccache", + "mkdir -p /root/.config/sccache && touch /root/.config/sccache/config", "mkdir -p /tmp/sccache-cache", "export SCCACHE_DIR=/tmp/sccache-cache", "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", From b789577ab28721fb165ad22a7d1f6936fd61c180 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:00:15 +0000 Subject: [PATCH 087/225] v2 --- .github/workflows/packaging_wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a49ce158..dae706cb 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -60,6 +60,7 @@ jobs: #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_DIR: /tmp/sccache-cache + ACTIONS_CACHE_SERVICE_V2: "1" steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 @@ -201,7 +202,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 37ad2e7693b827f374c631a6827a20a7f72deb95 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:09:40 +0000 Subject: [PATCH 088/225] ACTIONS_CACHE_SERVICE_V2 --- .github/workflows/packaging_wheels.yml | 3 ++- pyproject.toml | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index dae706cb..ec669889 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -202,7 +202,8 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on SCCACHE_DIR=/tmp/sccache-cache ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} + # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" diff --git a/pyproject.toml b/pyproject.toml index d0c0eaf7..6ad451d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,7 +383,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL", "SCCACHE_DIR"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", @@ -405,9 +405,9 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_URL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2"] before-build = ["choco install sccache"] From df3b2b40ec257a50f68df6b2843bc22f2bd56e77 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:12:53 +0000 Subject: [PATCH 089/225] keep debugging --- .github/workflows/packaging_wheels.yml | 39 ++++---------------------- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index ec669889..da0f67b6 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -59,7 +59,7 @@ jobs: uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" - SCCACHE_DIR: /tmp/sccache-cache + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ matrix.python }}-${{ matrix.platform.arch }} ACTIONS_CACHE_SERVICE_V2: "1" steps: - name: Checkout DuckDB Python @@ -108,35 +108,15 @@ jobs: llvm echo "APT package installation completed" - - name: Test sccache on host - run: | - echo "Testing sccache GitHub Actions cache access on host..." - sccache --start-server - echo "Host environment variables:" - echo "ACTIONS_RESULTS_URL: ${ACTIONS_RESULTS_URL}" - echo "ACTIONS_RUNTIME_TOKEN: ${ACTIONS_RUNTIME_TOKEN:+SET}" - echo "SCCACHE_GHA_ENABLED: ${SCCACHE_GHA_ENABLED}" - echo "Initial sccache stats:" - sccache --show-stats - echo 'int main(){return 0;}' > test_host.c - echo "Compiling test file with sccache on host..." - SCCACHE_LOG=debug sccache gcc -c test_host.c -o test_host.o 2>&1 | head -20 || true - echo "Final sccache stats on host:" - sccache --show-stats - rm -f test_host.c test_host.o - echo "Host sccache test complete" - env: - SCCACHE_GHA_ENABLED: "on" - - name: Restore Build Cache - uses: actions/cache/restore@v4 + + - name: Cache + uses: actions/cache@v4 with: path: | build/ wheelhouse/ /home/runner/.cache/cibuildwheel key: build-dir-${{ matrix.python }}-${{ runner.os }}- - restore-keys: | - build-dir-${{ matrix.python }}-${{ runner.os }}- - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -202,8 +182,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_GHA_ENABLED=on ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} - # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" @@ -220,12 +199,4 @@ jobs: run: | echo "Final sccache statistics on host:" sccache --show-stats || echo "sccache not available on host" - - name: Save Build Cache - if: always() # Always save, even on failure - uses: actions/cache/save@v4 - with: - path: | - build/ - wheelhouse/ - /home/runner/.cache/cibuildwheel key: build-dir-${{ matrix.python }}-${{ runner.os }}-e \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6ad451d7..4de92a52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,7 +383,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER"] before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", From 068fadda423217883262f6b83b97e29fe8e4dfa6 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:16:52 +0000 Subject: [PATCH 090/225] disable concurrency --- .github/workflows/packaging.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 755b86c5..ea078376 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -52,9 +52,9 @@ on: required: false type: string -concurrency: - group: packaging-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: packaging-${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true defaults: run: From d4d686641ea50d93a8f9031693a063010fbdb7f1 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:21:35 +0000 Subject: [PATCH 091/225] Disable cache --- .github/workflows/packaging_wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index da0f67b6..1d73db78 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -182,7 +182,8 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From f84dd56ddb87b32e740f876030e8fd75bb8fd2b7 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:23:18 +0000 Subject: [PATCH 092/225] no sccache at top level --- .github/workflows/packaging_wheels.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 1d73db78..afe0a526 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -194,10 +194,4 @@ jobs: with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl - compression-level: 0 - - name: Show final sccache stats - if: always() - run: | - echo "Final sccache statistics on host:" - sccache --show-stats || echo "sccache not available on host" - key: build-dir-${{ matrix.python }}-${{ runner.os }}-e \ No newline at end of file + compression-level: 0 \ No newline at end of file From 60abe5c33839ff675c9160a851cc380a1bfb589c Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:24:47 +0000 Subject: [PATCH 093/225] eanble for ubuntu --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index afe0a526..66dc5180 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,9 +34,9 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: - - { os: self-hosted, arch: x86_64, cibw_system: manylinux} + # - { os: self-hosted, arch: x86_64, cibw_system: manylinux} #- { os: windows-2025, arch: amd64, cibw_system: win } - #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } From c5f1b6a9c8a7f567803dd802169e6f5d1ea83ea2 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:31:35 +0000 Subject: [PATCH 094/225] emit versions --- .github/workflows/packaging_wheels.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 66dc5180..29af45db 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -59,7 +59,7 @@ jobs: uv run python -c "import duckdb" #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" - SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ matrix.python }}-${{ matrix.platform.arch }} + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" steps: - name: Checkout DuckDB Python @@ -108,6 +108,27 @@ jobs: llvm echo "APT package installation completed" + - name: Debug - Check compiler versions + run: | + echo "=== Runner Environment Debug ===" + echo "Runner OS: ${{ runner.os }}" + echo "Runner architecture: ${{ runner.arch }}" + echo "OS Info:" + cat /etc/os-release + echo "=== Compiler Versions ===" + gcc --version || echo "gcc not found" + g++ --version || echo "g++ not found" + clang --version || echo "clang not found" + clang++ --version || echo "clang++ not found" + echo "=== Available GCC Versions ===" + ls -la /usr/bin/gcc* || echo "no gcc binaries found" + ls -la /usr/bin/g++* || echo "no g++ binaries found" + echo "=== Build Environment ===" + which cmake && cmake --version || echo "cmake not found" + which ninja && ninja --version || echo "ninja not found" + echo "=== Python Environment ===" + which python && python --version || echo "python not found" + echo "================================" - name: Cache uses: actions/cache@v4 From 8101d105b3c5f78a018f07f42ae9217b52396e8e Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:31:48 +0000 Subject: [PATCH 095/225] emit versions self hosted --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 29af45db..f9fe3b4a 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,9 +34,9 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: - # - { os: self-hosted, arch: x86_64, cibw_system: manylinux} + - { os: self-hosted, arch: x86_64, cibw_system: manylinux} #- { os: windows-2025, arch: amd64, cibw_system: win } - - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + # - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } From 175cac4c8602e33143c355759883a89fc78ccac8 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:39:43 +0000 Subject: [PATCH 096/225] remove unneeded stuff --- .github/workflows/packaging_wheels.yml | 27 ++------------------------ 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index f9fe3b4a..c86bf016 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,9 +34,9 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: - - { os: self-hosted, arch: x86_64, cibw_system: manylinux} + # - { os: self-hosted, arch: x86_64, cibw_system: manylinux} #- { os: windows-2025, arch: amd64, cibw_system: win } - # - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } @@ -107,29 +107,6 @@ jobs: clang \ llvm echo "APT package installation completed" - - - name: Debug - Check compiler versions - run: | - echo "=== Runner Environment Debug ===" - echo "Runner OS: ${{ runner.os }}" - echo "Runner architecture: ${{ runner.arch }}" - echo "OS Info:" - cat /etc/os-release - echo "=== Compiler Versions ===" - gcc --version || echo "gcc not found" - g++ --version || echo "g++ not found" - clang --version || echo "clang not found" - clang++ --version || echo "clang++ not found" - echo "=== Available GCC Versions ===" - ls -la /usr/bin/gcc* || echo "no gcc binaries found" - ls -la /usr/bin/g++* || echo "no g++ binaries found" - echo "=== Build Environment ===" - which cmake && cmake --version || echo "cmake not found" - which ninja && ninja --version || echo "ninja not found" - echo "=== Python Environment ===" - which python && python --version || echo "python not found" - echo "================================" - - name: Cache uses: actions/cache@v4 with: From 326e8462d4e4f38c7ab840c2d39f975f23f55943 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 12:58:35 +0000 Subject: [PATCH 097/225] back to test --- .github/workflows/packaging_wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c86bf016..62bd96f8 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,8 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run python -c "import duckdb" - #uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" From a449c732ed41865ae387654ef700dd5c2c98c954 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 13:31:05 +0000 Subject: [PATCH 098/225] make tests pass --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 62bd96f8..5c875bf8 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" From a2a5cf8b5631c3c15ec2a94bfdc2b548975bd01a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 13:46:49 +0000 Subject: [PATCH 099/225] use nightly pandas --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4de92a52..211eeb14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() "pandas; python_version < '3.14'", # used for pandas dataframes all over the place + "pandas>=2.3.0.dev0; python_version >= '3.14' --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple", # nightly build for Python 3.14 "pyarrow; python_version < '3.14'", # used for pyarrow support "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package) ] @@ -243,6 +244,7 @@ test = [ # dependencies used for running tests "urllib3", "fsspec>=2022.11.0", "pandas>=2.0.0; python_version < '3.14'", + "pandas>=2.3.0.dev0; python_version >= '3.14' --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple", "pyarrow>=18.0.0; python_version < '3.14'", "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", @@ -258,6 +260,7 @@ scripts = [ # dependencies used for running scripts "ipywidgets", "numpy", "pandas; python_version < '3.14'", + "pandas>=2.3.0.dev0; python_version >= '3.14' --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple", "pcpp", "polars; python_version < '3.14'", "pyarrow; python_version < '3.14'", From 59713ea524726968e8e0418916f2469c1505e40a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 13:51:03 +0000 Subject: [PATCH 100/225] try 2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 211eeb14..b69ea34b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() "pandas; python_version < '3.14'", # used for pandas dataframes all over the place - "pandas>=2.3.0.dev0; python_version >= '3.14' --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple", # nightly build for Python 3.14 + "pandas>=2.3.0.dev0; python_version >= '3.14'", # nightly build for Python 3.14 "pyarrow; python_version < '3.14'", # used for pyarrow support "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package) ] From 735f5afb572f57170c93239b963b89ef5896c8a4 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 13:51:51 +0000 Subject: [PATCH 101/225] move index --- pyproject.toml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b69ea34b..6bdda3cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -214,9 +214,18 @@ name = "pytorch-cpu" url = "https://download.pytorch.org/whl/cpu" explicit = true +[[tool.uv.index]] +name = "scientific-python-nightly" +url = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" +explicit = true + [tool.uv.sources] torch = [ { index = "pytorch-cpu" } ] torchvision = [ { index = "pytorch-cpu" } ] +pandas = [ + { marker = "python_version < '3.14'" }, + { marker = "python_version >= '3.14'", index = "scientific-python-nightly" } +] [dependency-groups] # used for development only, requires pip >=25.1.0 stubdeps = [ # dependencies used for typehints in the stubs @@ -244,7 +253,7 @@ test = [ # dependencies used for running tests "urllib3", "fsspec>=2022.11.0", "pandas>=2.0.0; python_version < '3.14'", - "pandas>=2.3.0.dev0; python_version >= '3.14' --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple", + "pandas>=2.3.0.dev0; python_version >= '3.14'", "pyarrow>=18.0.0; python_version < '3.14'", "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", @@ -260,7 +269,7 @@ scripts = [ # dependencies used for running scripts "ipywidgets", "numpy", "pandas; python_version < '3.14'", - "pandas>=2.3.0.dev0; python_version >= '3.14' --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple", + "pandas>=2.3.0.dev0; python_version >= '3.14'", "pcpp", "polars; python_version < '3.14'", "pyarrow; python_version < '3.14'", From 8af90c604c7a6b2049bce5a3013e3317485caf9d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 14:06:05 +0000 Subject: [PATCH 102/225] update indices --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bdda3cd..f2a2da6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,10 +222,6 @@ explicit = true [tool.uv.sources] torch = [ { index = "pytorch-cpu" } ] torchvision = [ { index = "pytorch-cpu" } ] -pandas = [ - { marker = "python_version < '3.14'" }, - { marker = "python_version >= '3.14'", index = "scientific-python-nightly" } -] [dependency-groups] # used for development only, requires pip >=25.1.0 stubdeps = [ # dependencies used for typehints in the stubs From 6c3c84deb7c221a81b743dd89cdfb1f6ee367e89 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 14:25:19 +0000 Subject: [PATCH 103/225] accept new warning for pandas deprecation --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f2a2da6a..c70108be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -314,6 +314,8 @@ filterwarnings = [ # Pyspark is throwing these warnings "ignore:distutils Version classes are deprecated:DeprecationWarning", "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning", + # Pandas nightly build ChainedAssignmentError warnings + "ignore:ChainedAssignmentError.*:FutureWarning", ] [tool.coverage.run] From 0ac07f4c0feab1858e385d0d2a142e7f0ea9f28a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 14:57:43 +0000 Subject: [PATCH 104/225] keep testing locally --- .github/workflows/cleanup_pypi.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/packaging_sdist.yml | 2 +- .github/workflows/packaging_wheels.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cleanup_pypi.yml b/.github/workflows/cleanup_pypi.yml index c4300be3..e290faae 100644 --- a/.github/workflows/cleanup_pypi.yml +++ b/.github/workflows/cleanup_pypi.yml @@ -52,7 +52,7 @@ jobs: - name: Install Astral UV uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" - name: Run Cleanup env: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fdd2a838..ab696897 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -70,7 +70,7 @@ jobs: - name: Install Astral UV and enable the cache uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" python-version: 3.9 enable-cache: true cache-suffix: -${{ github.workflow }} diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index 2723b437..87923f4c 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -58,7 +58,7 @@ jobs: - name: Install Astral UV uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" python-version: 3.11 - name: Build sdist diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5c875bf8..86f65f27 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,7 +34,7 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: - # - { os: self-hosted, arch: x86_64, cibw_system: manylinux} + - { os: self-hosted, arch: x86_64, cibw_system: manylinux} #- { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } From f762785c7471b5abe75b2c56afa50baa6fb9f48b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 15:02:46 +0000 Subject: [PATCH 105/225] on my own --- .github/workflows/on_pr.yml | 2 ++ .github/workflows/packaging.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 7e4b5107..10e0abbf 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -19,6 +19,8 @@ on: # cancel-in-progress: true jobs: + runs-on: self-hosted + submodule_sanity_guard: name: Make sure submodule is in a sane state uses: ./.github/workflows/submodule_sanity.yml diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index ea078376..6bfb2bc1 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -61,6 +61,8 @@ defaults: shell: bash jobs: + runs-on: self-hosted + build_sdist: name: Build an sdist and determine versions if: ${{ github.event_name != 'pull_request' }} From 03bb7e94b3c8cab59920aa768cbe8fa466850c05 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 15:03:26 +0000 Subject: [PATCH 106/225] runs-on: self-hosted --- .github/workflows/on_pr.yml | 7 ++++++- .github/workflows/packaging.yml | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 10e0abbf..9d21d158 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -19,13 +19,16 @@ on: # cancel-in-progress: true jobs: - runs-on: self-hosted submodule_sanity_guard: + runs-on: self-hosted + name: Make sure submodule is in a sane state uses: ./.github/workflows/submodule_sanity.yml packaging_test: + runs-on: self-hosted + name: Build a minimal set of packages and run all tests on them needs: submodule_sanity_guard # Skip packaging tests for draft PRs @@ -37,6 +40,8 @@ jobs: duckdb-sha: ${{ github.base_ref }} coverage_test: + runs-on: self-hosted + name: Run coverage tests needs: submodule_sanity_guard # Only run coverage test for draft PRs diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 6bfb2bc1..5b20eb72 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -61,9 +61,10 @@ defaults: shell: bash jobs: - runs-on: self-hosted build_sdist: + runs-on: self-hosted + name: Build an sdist and determine versions if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/packaging_sdist.yml @@ -74,6 +75,8 @@ jobs: set-version: ${{ inputs.stable-version }} build_wheels: + runs-on: self-hosted + name: Build and test releases uses: ./.github/workflows/packaging_wheels.yml with: From 0ff34f905762da434799b41b79bdc7c76c3c28b5 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 15:07:03 +0000 Subject: [PATCH 107/225] dont r --- .github/workflows/on_pr.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 9d21d158..0876bebd 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -21,13 +21,11 @@ on: jobs: submodule_sanity_guard: - runs-on: self-hosted name: Make sure submodule is in a sane state uses: ./.github/workflows/submodule_sanity.yml packaging_test: - runs-on: self-hosted name: Build a minimal set of packages and run all tests on them needs: submodule_sanity_guard @@ -40,7 +38,6 @@ jobs: duckdb-sha: ${{ github.base_ref }} coverage_test: - runs-on: self-hosted name: Run coverage tests needs: submodule_sanity_guard From 633e887f87ee717411434f27018feec4f5bbc241 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 15:09:09 +0000 Subject: [PATCH 108/225] nope --- .github/workflows/on_pr.yml | 4 ---- .github/workflows/packaging.yml | 5 ----- 2 files changed, 9 deletions(-) diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 0876bebd..7e4b5107 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -19,14 +19,11 @@ on: # cancel-in-progress: true jobs: - submodule_sanity_guard: - name: Make sure submodule is in a sane state uses: ./.github/workflows/submodule_sanity.yml packaging_test: - name: Build a minimal set of packages and run all tests on them needs: submodule_sanity_guard # Skip packaging tests for draft PRs @@ -38,7 +35,6 @@ jobs: duckdb-sha: ${{ github.base_ref }} coverage_test: - name: Run coverage tests needs: submodule_sanity_guard # Only run coverage test for draft PRs diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 5b20eb72..ea078376 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -61,10 +61,7 @@ defaults: shell: bash jobs: - build_sdist: - runs-on: self-hosted - name: Build an sdist and determine versions if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/packaging_sdist.yml @@ -75,8 +72,6 @@ jobs: set-version: ${{ inputs.stable-version }} build_wheels: - runs-on: self-hosted - name: Build and test releases uses: ./.github/workflows/packaging_wheels.yml with: From 2be590fc8f72aac335cedf8609823e7301341b51 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 15:33:14 +0000 Subject: [PATCH 109/225] cache uv and exclude warnings in CIBW_TEST_COMMAND --- .github/workflows/packaging_wheels.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 86f65f27..c57f4dab 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs -W "ignore:ChainedAssignmentError.*:FutureWarning" SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" @@ -113,7 +113,8 @@ jobs: build/ wheelhouse/ /home/runner/.cache/cibuildwheel - key: build-dir-${{ matrix.python }}-${{ runner.os }}- + ${{ runner.home }}/.cache/uv + key: build-dir-${{ matrix.python }}-${{ runner.os }}-${{ hashFiles('**/pyproject.toml') }} - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -179,7 +180,8 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_LINUX: UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | From f33bce4a763d85b1f3aeb6eb054cadfc612adab9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 15:36:44 +0000 Subject: [PATCH 110/225] use a uv cache dir --- .github/workflows/packaging_wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c57f4dab..73011bb3 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -180,8 +180,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_LINUX: UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | From f93f112a1be0c2af7b75c2943ee82df411e4f43f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 15:59:40 +0000 Subject: [PATCH 111/225] quoting --- .github/workflows/packaging_wheels.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 73011bb3..4fae6982 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs -W "ignore:ChainedAssignmentError.*:FutureWarning" + PYTHON_GIL=1 PYTHONWARNINGS="ignore:ChainedAssignmentError.*:FutureWarning" uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" @@ -110,9 +110,6 @@ jobs: uses: actions/cache@v4 with: path: | - build/ - wheelhouse/ - /home/runner/.cache/cibuildwheel ${{ runner.home }}/.cache/uv key: build-dir-${{ matrix.python }}-${{ runner.os }}-${{ hashFiles('**/pyproject.toml') }} - name: Checkout DuckDB From bc925c7a4a723b86a6d1669c73ac31d584f70db9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 16:02:58 +0000 Subject: [PATCH 112/225] Move filter earlier --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 4fae6982..32c85ec9 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - PYTHON_GIL=1 PYTHONWARNINGS="ignore:ChainedAssignmentError.*:FutureWarning" uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" diff --git a/pyproject.toml b/pyproject.toml index c70108be..f07593c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -307,15 +307,15 @@ minversion = "6.0" addopts = "-ra -q" testpaths = ["tests"] filterwarnings = [ + # Pandas nightly build ChainedAssignmentError warnings + "ignore:ChainedAssignmentError.*:FutureWarning", "error", "ignore::UserWarning", # Jupyter is throwing DeprecationWarnings "ignore:function ham() is deprecated:DeprecationWarning", # Pyspark is throwing these warnings "ignore:distutils Version classes are deprecated:DeprecationWarning", - "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning", - # Pandas nightly build ChainedAssignmentError warnings - "ignore:ChainedAssignmentError.*:FutureWarning", + "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning" ] [tool.coverage.run] From 1625615060ccff79b190552f6d82600d7075fe10 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 16:19:14 +0000 Subject: [PATCH 113/225] reorder warnings --- .github/workflows/packaging_wheels.yml | 4 ++-- pyproject.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 32c85ec9..487cc781 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,7 +34,7 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: - - { os: self-hosted, arch: x86_64, cibw_system: manylinux} + #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} #- { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } @@ -56,7 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" diff --git a/pyproject.toml b/pyproject.toml index f07593c2..d6149143 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -415,9 +415,9 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER"] before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER"] before-build = ["choco install sccache"] From 73953f93a6e26cb1ae38958b8af9e938ef2e978f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 16:35:32 +0000 Subject: [PATCH 114/225] pass exclusion --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 487cc781..5e111e81 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + PYTHON_GIL=1 PYTHONWARNINGS="ignore:ChainedAssignmentError.*:FutureWarning" uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" From a6afd083cbb81e9be74c8ebef3a68582813fee82 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 17:53:34 +0000 Subject: [PATCH 115/225] it's an exception --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5e111e81..487cc781 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - PYTHON_GIL=1 PYTHONWARNINGS="ignore:ChainedAssignmentError.*:FutureWarning" uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" diff --git a/pyproject.toml b/pyproject.toml index d6149143..d2741bfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -307,8 +307,8 @@ minversion = "6.0" addopts = "-ra -q" testpaths = ["tests"] filterwarnings = [ - # Pandas nightly build ChainedAssignmentError warnings - "ignore:ChainedAssignmentError.*:FutureWarning", + # Pandas nightly build ChainedAssignmentError exceptions + "ignore::pandas.errors.ChainedAssignmentError", "error", "ignore::UserWarning", # Jupyter is throwing DeprecationWarnings From a60e5334c70a2d84159e8cd42b2d681e128891b0 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 17:58:56 +0000 Subject: [PATCH 116/225] Use the nightly wheel, don't rebuild --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d2741bfc..23ce9ed1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,6 +222,10 @@ explicit = true [tool.uv.sources] torch = [ { index = "pytorch-cpu" } ] torchvision = [ { index = "pytorch-cpu" } ] +pandas = [ + { marker = "python_version < '3.14'" }, + { marker = "python_version >= '3.14'", index = "scientific-python-nightly" } +] [dependency-groups] # used for development only, requires pip >=25.1.0 stubdeps = [ # dependencies used for typehints in the stubs From 53ad916ae471360095c75bd05f11226766ee344d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 18:08:52 +0000 Subject: [PATCH 117/225] set index --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 23ce9ed1..02bd125f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,8 +223,8 @@ explicit = true torch = [ { index = "pytorch-cpu" } ] torchvision = [ { index = "pytorch-cpu" } ] pandas = [ - { marker = "python_version < '3.14'" }, - { marker = "python_version >= '3.14'", index = "scientific-python-nightly" } + { index = "pypi", marker = "python_version < '3.14'" }, + { index = "scientific-python-nightly", marker = "python_version >= '3.14'" } ] [dependency-groups] # used for development only, requires pip >=25.1.0 From 660ba3fc7cf5c3a1ea57722e80a73b72ae092967 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 18:16:01 +0000 Subject: [PATCH 118/225] default for <3.14 --- .github/workflows/packaging_wheels.yml | 7 +++++- pyproject.toml | 1 - test_mre.py | 11 +++++++++ test_warnings_mre.py | 31 ++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 test_mre.py create mode 100644 test_warnings_mre.py diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 487cc781..cd2c5bcc 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -173,7 +173,12 @@ jobs: run: python -m pip install cibuildwheel==3.1.4 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - run: python -m cibuildwheel --output-dir wheelhouse + run: | + echo "Testing dependency installation without build..." + export UV_CACHE_DIR="${{ runner.home }}/.cache/uv" + uv export --only-group test --no-emit-project --output-file pylock.toml + uv pip install -r pylock.toml + echo "Dependencies installed successfully" env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} diff --git a/pyproject.toml b/pyproject.toml index 02bd125f..137848ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,7 +223,6 @@ explicit = true torch = [ { index = "pytorch-cpu" } ] torchvision = [ { index = "pytorch-cpu" } ] pandas = [ - { index = "pypi", marker = "python_version < '3.14'" }, { index = "scientific-python-nightly", marker = "python_version >= '3.14'" } ] diff --git a/test_mre.py b/test_mre.py new file mode 100644 index 00000000..99a2414b --- /dev/null +++ b/test_mre.py @@ -0,0 +1,11 @@ +import pandas as pd + + +def test_chained_assignment_warning(): + """This should trigger the ChainedAssignmentError warning we're trying to suppress""" + df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]}) + + # This chained assignment triggers the warning + df['A'][0] = 999 + + assert df['A'][0] == 999 \ No newline at end of file diff --git a/test_warnings_mre.py b/test_warnings_mre.py new file mode 100644 index 00000000..f0eef295 --- /dev/null +++ b/test_warnings_mre.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +""" +Minimal reproducible example for pandas ChainedAssignmentError warning filtering +""" +import warnings +import pandas as pd + + +def test_chained_assignment_warning(): + """Test that reproduces the ChainedAssignmentError warning from pandas nightly""" + # This should trigger the warning we're trying to suppress + df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]}) + + # This chained assignment should trigger the ChainedAssignmentError warning + df['A'][0] = 999 + + print("Test completed - if you see a FutureWarning above, the filter isn't working") + + +if __name__ == "__main__": + print("=== Running without any warning filters ===") + test_chained_assignment_warning() + + print("\n=== Running with PYTHONWARNINGS filter ===") + import os + os.environ['PYTHONWARNINGS'] = 'ignore:ChainedAssignmentError.*:FutureWarning' + test_chained_assignment_warning() + + print("\n=== Running with warnings.filterwarnings ===") + warnings.filterwarnings('ignore', message='ChainedAssignmentError.*', category=FutureWarning) + test_chained_assignment_warning() \ No newline at end of file From 223f03a5487e2d1958a736c49d80694b83491915 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 18:21:07 +0000 Subject: [PATCH 119/225] undo --- .github/workflows/packaging_wheels.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index cd2c5bcc..487cc781 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -173,12 +173,7 @@ jobs: run: python -m pip install cibuildwheel==3.1.4 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - run: | - echo "Testing dependency installation without build..." - export UV_CACHE_DIR="${{ runner.home }}/.cache/uv" - uv export --only-group test --no-emit-project --output-file pylock.toml - uv pip install -r pylock.toml - echo "Dependencies installed successfully" + run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} From d3acfb3842e63a37389d2de636f84e9e5a94cbf1 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 18:43:09 +0000 Subject: [PATCH 120/225] drop back a version --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 137848ad..d2741bfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,9 +222,6 @@ explicit = true [tool.uv.sources] torch = [ { index = "pytorch-cpu" } ] torchvision = [ { index = "pytorch-cpu" } ] -pandas = [ - { index = "scientific-python-nightly", marker = "python_version >= '3.14'" } -] [dependency-groups] # used for development only, requires pip >=25.1.0 stubdeps = [ # dependencies used for typehints in the stubs From a64e156cfa75c3b63da3107a7a7aa5adf465af7e Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 19:07:26 +0000 Subject: [PATCH 121/225] populate cache --- .github/workflows/packaging_wheels.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 487cc781..61542534 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -32,15 +32,15 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t + python: [ cp313, cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t platform: #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} - #- { os: windows-2025, arch: amd64, cibw_system: win } + - { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - #- { os: macos-15, arch: arm64, cibw_system: macosx } - #- { os: macos-15, arch: universal2, cibw_system: macosx } - #- { os: macos-13, arch: x86_64, cibw_system: macosx } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + - { os: macos-15, arch: arm64, cibw_system: macosx } + - { os: macos-15, arch: universal2, cibw_system: macosx } + - { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: From 0231b7dfe102e7ebb6a5ea6f6cdf05a73fb5c9a6 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 19:12:41 +0000 Subject: [PATCH 122/225] fix env --- .github/workflows/packaging_wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 61542534..1e143550 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -177,7 +177,8 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_LINUX: UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | From 6a086cc22f448cdec479237c00be7ee9c4f53869 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 19:20:38 +0000 Subject: [PATCH 123/225] building quickly --- .github/workflows/packaging_wheels.yml | 13 ------------- pyproject.toml | 5 +++-- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 1e143550..ce18c569 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -141,19 +141,6 @@ jobs: fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] }} - - name: Verify Python version - run: | - expected="${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] }}" - actual=$(python --version | cut -d' ' -f2) - echo "Expected: $expected, Actual: $actual" - if [[ "$expected" == "3.14t" && "$actual" =~ ^3\.14\. ]]; then - echo "Python 3.14t version check passed" - elif [[ "$actual" =~ ^$expected\. ]]; then - echo "Python version check passed" - else - echo "Python version mismatch!" - exit 1 - fi - name: Debug Python paths run: | diff --git a/pyproject.toml b/pyproject.toml index d2741bfc..a579ca9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -395,8 +395,9 @@ enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER"] before-build = [ - "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz | tar xz", - "cp sccache-v0.10.0-x86_64-unknown-linux-musl/sccache /usr/bin", + "if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCH=aarch64; else ARCH=x86_64; fi", + "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz", + "cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin", "which sccache", "mkdir -p /root/.config/sccache && touch /root/.config/sccache/config", "mkdir -p /tmp/sccache-cache", From e807a9057d59a9798ceb0c6e63d53bb5ca0c41ca Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 19:23:01 +0000 Subject: [PATCH 124/225] remove debug --- .github/workflows/packaging_wheels.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index ce18c569..101e1b91 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -142,19 +142,6 @@ jobs: }} - - name: Debug Python paths - run: | - echo "Current python: $(which python)" - echo "Python version: $(python --version)" - echo "pythonLocation: $pythonLocation" - echo "Checking CIBW_TOOL_PYTHON path: /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python" - ls -la /opt/hostedtoolcache/Python/3.14.0-rc.2/x64-freethreaded/bin/python || echo "Path not found" - - - name: Debug CIBW_TOOL_PYTHON path - run: | - echo "pythonLocation: $pythonLocation" - echo "Expected CIBW_TOOL_PYTHON: $pythonLocation/bin/python" - ls -la "$pythonLocation/bin/python" || echo "Python not found at expected location" - name: Install cibuildwheel run: python -m pip install cibuildwheel==3.1.4 From bd0f564cda3e5b22b541ab5db6fe94c44b098d32 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 19:32:34 +0000 Subject: [PATCH 125/225] fix sccache for the millionth time --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 101e1b91..2413d3ad 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -151,7 +151,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 From 73d99f2e76499cc04cec8046faf14102d2c486fc Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 19:56:34 +0000 Subject: [PATCH 126/225] put the warning --- pyproject.toml | 5 ++--- tests/pytest.ini | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a579ca9e..ed1118ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -307,15 +307,14 @@ minversion = "6.0" addopts = "-ra -q" testpaths = ["tests"] filterwarnings = [ - # Pandas nightly build ChainedAssignmentError exceptions - "ignore::pandas.errors.ChainedAssignmentError", "error", "ignore::UserWarning", # Jupyter is throwing DeprecationWarnings "ignore:function ham() is deprecated:DeprecationWarning", # Pyspark is throwing these warnings "ignore:distutils Version classes are deprecated:DeprecationWarning", - "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning" + "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning", + "ignore:ChainedAssignmentError.*:FutureWarning" ] [tool.coverage.run] diff --git a/tests/pytest.ini b/tests/pytest.ini index 0c17afd5..5081ee33 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -2,6 +2,8 @@ [pytest] filterwarnings = error + # Pandas ChainedAssignmentError warnings for 3.0 + ignore:ChainedAssignmentError.*:FutureWarning ignore::UserWarning ignore::DeprecationWarning # Jupyter is throwing DeprecationWarnings From 24d7013bbba88e1f855301c7816028ca5598cb79 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 20:01:39 +0000 Subject: [PATCH 127/225] avoid sccache error --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 2413d3ad..101e1b91 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -151,7 +151,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} + CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CIBW_ENVIRONMENT_LINUX: UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 From 75172d2122b82855a9557b778596d3b0caa52435 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 20:05:25 +0000 Subject: [PATCH 128/225] remove the linux entry --- .github/workflows/packaging_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 101e1b91..a5f4e4d2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -152,7 +152,6 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_LINUX: UV_CACHE_DIR=/host${{ runner.home }}/.cache/uv # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | From 26918751ab1eeec2da2cd9486c92c388b954d5fb Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 20:53:34 +0000 Subject: [PATCH 129/225] speed up windows --- .github/workflows/packaging_wheels.yml | 13 +++++++------ pyproject.toml | 14 +++++++++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a5f4e4d2..4fdce907 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -36,11 +36,11 @@ jobs: platform: #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} - { os: windows-2025, arch: amd64, cibw_system: win } - - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - - { os: macos-15, arch: arm64, cibw_system: macosx } - - { os: macos-15, arch: universal2, cibw_system: macosx } - - { os: macos-13, arch: x86_64, cibw_system: macosx } + #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + #- { os: macos-15, arch: arm64, cibw_system: macosx } + #- { os: macos-15, arch: universal2, cibw_system: macosx } + #- { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: @@ -56,10 +56,11 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - PYTHON_GIL=1 uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" + PYTHON_GIL: "1" steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 diff --git a/pyproject.toml b/pyproject.toml index ed1118ed..81e78e19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -392,7 +392,7 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ "if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCH=aarch64; else ARCH=x86_64; fi", "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz", @@ -415,9 +415,17 @@ before-build = [ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +environment = { CMAKE_C_COMPILER_LAUNCHER = "sccache", CMAKE_CXX_COMPILER_LAUNCHER = "sccache" } before-build = ["brew install sccache"] +after-build = ["sccache --show-stats"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +environment = { CMAKE_C_COMPILER_LAUNCHER = "sccache", CMAKE_CXX_COMPILER_LAUNCHER = "sccache" } before-build = ["choco install sccache"] +after-build = ["sccache --show-stats"] + +[[tool.cibuildwheel.overrides]] +select = "cp314t-*" +environment = { Python_LIBRARY_RELEASE = "python314t.lib" } From c1ba9bda1ef2609390e400b3b6bb0cdf4e11eae8 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 20:59:42 +0000 Subject: [PATCH 130/225] before-test --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81e78e19..2bcadb6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -418,14 +418,14 @@ before-test = "sccache --show-stats" environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] environment = { CMAKE_C_COMPILER_LAUNCHER = "sccache", CMAKE_CXX_COMPILER_LAUNCHER = "sccache" } before-build = ["brew install sccache"] -after-build = ["sccache --show-stats"] +before-test = ["sccache --show-stats"] [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] environment = { CMAKE_C_COMPILER_LAUNCHER = "sccache", CMAKE_CXX_COMPILER_LAUNCHER = "sccache" } before-build = ["choco install sccache"] -after-build = ["sccache --show-stats"] +before-test = ["sccache --show-stats"] [[tool.cibuildwheel.overrides]] select = "cp314t-*" -environment = { Python_LIBRARY_RELEASE = "python314t.lib" } +environment = { Python_LIBRARY_RELEASE = "python314t.lib" } \ No newline at end of file From bf80e77055b759497571c50698614ecb2e126b39 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 21:08:16 +0000 Subject: [PATCH 131/225] sccache --- .github/workflows/packaging_wheels.yml | 13 ++++++++----- pyproject.toml | 2 -- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 4fdce907..06ccbc8e 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -68,10 +68,11 @@ jobs: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - - name: Setup sccache # idea from https://github.com/pypa/cibuildwheel/issues/1030 - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: "v0.10.0" + # The host doesn't need sccache, only the cibuildwheel container + # - name: Setup sccache # idea from https://github.com/pypa/cibuildwheel/issues/1030 + # uses: mozilla-actions/sccache-action@v0.0.9 + # with: + # version: "v0.10.0" - name: Configure Cache Env uses: actions/github-script@v7 with: @@ -152,7 +153,9 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | diff --git a/pyproject.toml b/pyproject.toml index 2bcadb6a..11f2f41c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -416,13 +416,11 @@ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] -environment = { CMAKE_C_COMPILER_LAUNCHER = "sccache", CMAKE_CXX_COMPILER_LAUNCHER = "sccache" } before-build = ["brew install sccache"] before-test = ["sccache --show-stats"] [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] -environment = { CMAKE_C_COMPILER_LAUNCHER = "sccache", CMAKE_CXX_COMPILER_LAUNCHER = "sccache" } before-build = ["choco install sccache"] before-test = ["sccache --show-stats"] From bc43b83d8aa3d28668e32cf5de127c380f8e7cd3 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 21:13:55 +0000 Subject: [PATCH 132/225] sccache logging --- .github/workflows/packaging_wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 06ccbc8e..9a8b9786 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -153,9 +153,9 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | From ffc6251f364bad792808012bd1c0c1ab668df01d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 21:24:17 +0000 Subject: [PATCH 133/225] sccache stats --- .github/workflows/packaging_wheels.yml | 1 + pyproject.toml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9a8b9786..41e6fba5 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,6 +53,7 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > + sccache --show-stats uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > diff --git a/pyproject.toml b/pyproject.toml index 11f2f41c..c6111a61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -417,7 +417,6 @@ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = ["brew install sccache"] -before-test = ["sccache --show-stats"] [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] From 7b522a69c5e44aec155e35800630ead5f510119f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 22:19:16 +0000 Subject: [PATCH 134/225] not one line --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 41e6fba5..598f8d25 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,7 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - sccache --show-stats + sccache --show-stats && uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > From 5b8fdb76d2b226da1ab418f095ca3a73e5791fb5 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 23:10:27 +0000 Subject: [PATCH 135/225] pass as cmake arg --- .github/workflows/packaging_wheels.yml | 6 +++--- pyproject.toml | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 598f8d25..337cbae2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp313, cp314t ] # cp39, cp310, cp311, cp312, cp314, cp314t + python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} - { os: windows-2025, arch: amd64, cibw_system: win } @@ -53,7 +53,7 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - sccache --show-stats && + sccache --show-stats &&PYTHON_GIL=1 uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > @@ -155,7 +155,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER="${{ toJSON(matrix) }}" PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 diff --git a/pyproject.toml b/pyproject.toml index c6111a61..f8b0c15e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -424,5 +424,7 @@ before-build = ["choco install sccache"] before-test = ["sccache --show-stats"] [[tool.cibuildwheel.overrides]] +# Use the correct lib file for 3.14t, without this it looks +# for python314.lib select = "cp314t-*" -environment = { Python_LIBRARY_RELEASE = "python314t.lib" } \ No newline at end of file +environment = { CMAKE_ARGS = "-DPython_LIBRARY_RELEASE=python314t.lib" } \ No newline at end of file From 017f19f0d99be16c34a28024751234d94344c9ac Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 23:14:37 +0000 Subject: [PATCH 136/225] sccache --- .github/workflows/packaging_wheels.yml | 13 ++++++++++++- pyproject.toml | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 337cbae2..5f9b0d01 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,7 +155,18 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER="${{ toJSON(matrix) }}" PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: > + SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} + CMAKE_C_COMPILER_LAUNCHER=sccache + CMAKE_CXX_COMPILER_LAUNCHER=sccache + SCCACHE_LOG=debug + SCCACHE_NO_DAEMON=1 + SCCACHE_GHA_ENABLED=on + ACTIONS_RUNTIME_TOKEN=${{ github.token }} + ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} + ACTIONS_CACHE_SERVICE_V2=1 + SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} + PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 diff --git a/pyproject.toml b/pyproject.toml index f8b0c15e..ddb9a346 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -420,8 +420,21 @@ before-build = ["brew install sccache"] [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] -before-build = ["choco install sccache"] -before-test = ["sccache --show-stats"] +before-build = [ + "choco install sccache", + "echo 'sccache installation complete, verifying...'", + "sccache --version", + "sccache --show-stats", + "echo 'CMAKE_C_COMPILER_LAUNCHER will be set to: sccache'", + "echo 'CMAKE_CXX_COMPILER_LAUNCHER will be set to: sccache'", + "echo 'SCCACHE_LOG=debug - debug logging enabled'", + "echo 'SCCACHE_GHA_ENABLED=on - GitHub Actions cache enabled'", + "echo 'Starting build with sccache...'" +] +before-test = [ + "echo 'Final sccache statistics:'", + "sccache --show-stats" +] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From 507e7ee5e64f08dde6b922d49cc918dd47aa7767 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 23:21:17 +0000 Subject: [PATCH 137/225] passing sccache --- .github/workflows/packaging_wheels.yml | 13 +------------ pyproject.toml | 1 - 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5f9b0d01..9d93ec39 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,18 +155,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: > - SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} - CMAKE_C_COMPILER_LAUNCHER=sccache - CMAKE_CXX_COMPILER_LAUNCHER=sccache - SCCACHE_LOG=debug - SCCACHE_NO_DAEMON=1 - SCCACHE_GHA_ENABLED=on - ACTIONS_RUNTIME_TOKEN=${{ github.token }} - ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} - ACTIONS_CACHE_SERVICE_V2=1 - SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} - PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER=sccache CMAKE_CXX_COMPILER_LAUNCHER=sccache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 diff --git a/pyproject.toml b/pyproject.toml index ddb9a346..5c6deebe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -419,7 +419,6 @@ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RES before-build = ["brew install sccache"] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ "choco install sccache", "echo 'sccache installation complete, verifying...'", From 801606a270bfe52f882c95415c63f044567628f7 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 23:29:13 +0000 Subject: [PATCH 138/225] sccache --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9d93ec39..2f1475e3 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,7 +155,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER=sccache CMAKE_CXX_COMPILER_LAUNCHER=sccache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 From 37fbed1257c7f9681d126a3976478589e4d6e65f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 23:36:41 +0000 Subject: [PATCH 139/225] sccache before ccache --- pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c6deebe..0d3c4f5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -423,9 +423,13 @@ before-build = [ "choco install sccache", "echo 'sccache installation complete, verifying...'", "sccache --version", + "where sccache", + "where ccache", "sccache --show-stats", - "echo 'CMAKE_C_COMPILER_LAUNCHER will be set to: sccache'", - "echo 'CMAKE_CXX_COMPILER_LAUNCHER will be set to: sccache'", + "echo 'Adding sccache to front of PATH to override ccache...'", + "set PATH=C:\\ProgramData\\chocolatey\\lib\\sccache\\tools;%PATH%", + "where sccache", + "echo 'CMAKE will auto-detect sccache (should override ccache)'", "echo 'SCCACHE_LOG=debug - debug logging enabled'", "echo 'SCCACHE_GHA_ENABLED=on - GitHub Actions cache enabled'", "echo 'Starting build with sccache...'" From 82af19c56e01ffe70b1036efe7bc36fd25f3b15d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Thu, 11 Sep 2025 23:37:51 +0000 Subject: [PATCH 140/225] add direct path --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 2f1475e3..9c80a06c 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,7 +155,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/chocolatey/lib/sccache/tools/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/chocolatey/lib/sccache/tools/sccache.exe" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 From e1019adc4f5e2d9a76020c4bbfab5ea044e7eb21 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 00:07:07 +0000 Subject: [PATCH 141/225] sccache --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9c80a06c..9f3922e0 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,7 +155,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/chocolatey/lib/sccache/tools/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/chocolatey/lib/sccache/tools/sccache.exe" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 From 834790dc3b2241fd2f770f99ea49a27b725282b9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 00:15:00 +0000 Subject: [PATCH 142/225] again --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9f3922e0..6eccf235 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,7 +155,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CCACHE_PROGRAM="C:/ProgramData/Chocolatey/bin/sccache.exe" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 1 From 8666c2b7f54c823b1055174758d210cd0af5a70b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 00:28:59 +0000 Subject: [PATCH 143/225] test --- .github/workflows/packaging_wheels.yml | 4 ++-- pyproject.toml | 32 ++++++++++++++++++-------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 6eccf235..ab9a6a21 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,10 +155,10 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CCACHE_PROGRAM="C:/ProgramData/Chocolatey/bin/sccache.exe" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_BUILD_VERBOSITY: 1 + CIBW_BUILD_VERBOSITY: 3 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" sccache --show-stats diff --git a/pyproject.toml b/pyproject.toml index 0d3c4f5d..d1ccab10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -419,20 +419,34 @@ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RES before-build = ["brew install sccache"] [tool.cibuildwheel.windows] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ "choco install sccache", "echo 'sccache installation complete, verifying...'", "sccache --version", "where sccache", - "where ccache", - "sccache --show-stats", - "echo 'Adding sccache to front of PATH to override ccache...'", - "set PATH=C:\\ProgramData\\chocolatey\\lib\\sccache\\tools;%PATH%", - "where sccache", - "echo 'CMAKE will auto-detect sccache (should override ccache)'", - "echo 'SCCACHE_LOG=debug - debug logging enabled'", - "echo 'SCCACHE_GHA_ENABLED=on - GitHub Actions cache enabled'", - "echo 'Starting build with sccache...'" + "echo 'Verifying sccache executable path...'", + "dir \"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\"", + "\"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\" --version", + "echo 'Testing sccache GitHub Actions cache access...'", + "set SCCACHE_LOG=debug", + "sccache --show-stats", + "echo 'Testing sccache compilation with cl.exe...'", + "echo int main(){return 0;} > test.c", + "sccache cl.exe /c test.c /Fo:test.obj 2>&1 || echo 'sccache cl.exe test failed'", + "sccache --show-stats", + "if exist test.c del test.c", + "if exist test.obj del test.obj", + "echo 'Testing sccache compilation with gcc...'", + "echo int main(){return 0;} > test.c", + "sccache gcc -c test.c -o test.o 2>&1 || echo 'sccache gcc test failed'", + "sccache --show-stats", + "if exist test.c del test.c", + "if exist test.o del test.o", + "echo 'CMAKE compiler launcher verification:'", + "echo CMAKE_C_COMPILER_LAUNCHER=%CMAKE_C_COMPILER_LAUNCHER%", + "echo CMAKE_CXX_COMPILER_LAUNCHER=%CMAKE_CXX_COMPILER_LAUNCHER%", + "echo 'Sccache pre-build verification complete'" ] before-test = [ "echo 'Final sccache statistics:'", From 16e10fce0cbd3652c192d57f4d50dc152a9d9cf5 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 00:40:10 +0000 Subject: [PATCH 144/225] remove var, let win detect --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index ab9a6a21..96cfaf67 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -155,7 +155,7 @@ jobs: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_CXX_COMPILER_LAUNCHER="C:/ProgramData/Chocolatey/bin/sccache.exe" CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 3 From 0f3286b22c497b53b2950d82b60dcba23ececd3b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 00:48:28 +0000 Subject: [PATCH 145/225] debugging win --- .github/workflows/packaging_wheels.yml | 6 +++--- pyproject.toml | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 96cfaf67..6f7029b5 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -36,7 +36,7 @@ jobs: platform: #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} - { os: windows-2025, arch: amd64, cibw_system: win } - #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } @@ -154,9 +154,9 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="sccache" CMAKE_CXX_COMPILER_LAUNCHER="sccache" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_BUILD_VERBOSITY: 3 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | diff --git a/pyproject.toml b/pyproject.toml index d1ccab10..de29c8cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -398,6 +398,11 @@ before-build = [ "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz", "cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin", "which sccache", + "echo 'Compiler launcher detection order check:'", + "which ccache || echo 'ccache not found'", + "which sccache || echo 'sccache not found'", + "echo 'PATH priority test:'", + "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done", "mkdir -p /root/.config/sccache && touch /root/.config/sccache/config", "mkdir -p /tmp/sccache-cache", "export SCCACHE_DIR=/tmp/sccache-cache", @@ -416,15 +421,29 @@ before-test = "sccache --show-stats" [tool.cibuildwheel.macos] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] -before-build = ["brew install sccache"] +before-build = [ + "brew install sccache", + "echo 'Compiler launcher detection order check:'", + "which ccache || echo 'ccache not found'", + "which sccache || echo 'sccache not found'", + "echo 'PATH priority test:'", + "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done" +] [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ + "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", + "choco uninstall strawberryperl -y --force || echo 'Strawberry Perl not installed or removal failed'", "choco install sccache", "echo 'sccache installation complete, verifying...'", "sccache --version", "where sccache", + "echo 'Compiler launcher detection order check:'", + "where ccache 2>nul || echo 'ccache not found'", + "where sccache 2>nul || echo 'sccache not found'", + "echo 'PATH priority test:'", + "for %p in (ccache.exe sccache.exe) do @(where %p 2>nul && echo %p found) || echo %p not found", "echo 'Verifying sccache executable path...'", "dir \"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\"", "\"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\" --version", From 382b3059a517ec046ac020bc8ba43e0090868096 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 00:59:45 +0000 Subject: [PATCH 146/225] f2 --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 6f7029b5..cd9ff2bb 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -154,7 +154,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_PROGRAM="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 diff --git a/pyproject.toml b/pyproject.toml index de29c8cd..92e6630e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -434,16 +434,18 @@ before-build = [ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", - "choco uninstall strawberryperl -y --force || echo 'Strawberry Perl not installed or removal failed'", + "choco uninstall strawberryperl -y --force", "choco install sccache", + "echo 'Refreshing environment after sccache install...'", + "refreshenv", "echo 'sccache installation complete, verifying...'", "sccache --version", "where sccache", "echo 'Compiler launcher detection order check:'", - "where ccache 2>nul || echo 'ccache not found'", - "where sccache 2>nul || echo 'sccache not found'", + "where ccache 2>nul", + "where sccache 2>nul", "echo 'PATH priority test:'", - "for %p in (ccache.exe sccache.exe) do @(where %p 2>nul && echo %p found) || echo %p not found", + "for %p in (ccache.exe sccache.exe) do @where %p 2>nul", "echo 'Verifying sccache executable path...'", "dir \"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\"", "\"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\" --version", @@ -452,13 +454,13 @@ before-build = [ "sccache --show-stats", "echo 'Testing sccache compilation with cl.exe...'", "echo int main(){return 0;} > test.c", - "sccache cl.exe /c test.c /Fo:test.obj 2>&1 || echo 'sccache cl.exe test failed'", + "sccache cl.exe /c test.c /Fo:test.obj 2>&1", "sccache --show-stats", "if exist test.c del test.c", "if exist test.obj del test.obj", "echo 'Testing sccache compilation with gcc...'", "echo int main(){return 0;} > test.c", - "sccache gcc -c test.c -o test.o 2>&1 || echo 'sccache gcc test failed'", + "sccache gcc -c test.c -o test.o 2>&1", "sccache --show-stats", "if exist test.c del test.c", "if exist test.o del test.o", From f86fad2a3df7978a58747d1ca9b5fdb2f9b7dc0c Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 01:08:51 +0000 Subject: [PATCH 147/225] fix linux --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 33 +------------------------- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index cd9ff2bb..69a12c83 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -154,7 +154,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CCACHE_PROGRAM="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 diff --git a/pyproject.toml b/pyproject.toml index 92e6630e..e2399895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -436,38 +436,7 @@ before-build = [ "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", "choco uninstall strawberryperl -y --force", "choco install sccache", - "echo 'Refreshing environment after sccache install...'", - "refreshenv", - "echo 'sccache installation complete, verifying...'", - "sccache --version", - "where sccache", - "echo 'Compiler launcher detection order check:'", - "where ccache 2>nul", - "where sccache 2>nul", - "echo 'PATH priority test:'", - "for %p in (ccache.exe sccache.exe) do @where %p 2>nul", - "echo 'Verifying sccache executable path...'", - "dir \"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\"", - "\"C:\\ProgramData\\Chocolatey\\bin\\sccache.exe\" --version", - "echo 'Testing sccache GitHub Actions cache access...'", - "set SCCACHE_LOG=debug", - "sccache --show-stats", - "echo 'Testing sccache compilation with cl.exe...'", - "echo int main(){return 0;} > test.c", - "sccache cl.exe /c test.c /Fo:test.obj 2>&1", - "sccache --show-stats", - "if exist test.c del test.c", - "if exist test.obj del test.obj", - "echo 'Testing sccache compilation with gcc...'", - "echo int main(){return 0;} > test.c", - "sccache gcc -c test.c -o test.o 2>&1", - "sccache --show-stats", - "if exist test.c del test.c", - "if exist test.o del test.o", - "echo 'CMAKE compiler launcher verification:'", - "echo CMAKE_C_COMPILER_LAUNCHER=%CMAKE_C_COMPILER_LAUNCHER%", - "echo CMAKE_CXX_COMPILER_LAUNCHER=%CMAKE_CXX_COMPILER_LAUNCHER%", - "echo 'Sccache pre-build verification complete'" + "sccache --version" ] before-test = [ "echo 'Final sccache statistics:'", From 0fe2bf30bbee36f4ae9cc9ee11fed01e29022a53 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 01:25:30 +0000 Subject: [PATCH 148/225] win4 --- .github/workflows/packaging_wheels.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 69a12c83..8db4221a 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,7 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - sccache --show-stats &&PYTHON_GIL=1 + sccache --show-stats uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > @@ -154,11 +154,13 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_VS_GLOBALS="CLToolExe=sccache.exe;TrackFileAccess=false;UseMultiToolTask=true" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + # PYTHON_GIL=1 + # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - # SCCACHE_DIR=/tmp/sccache-cache SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_BUILD_VERBOSITY: 3 + CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" sccache --show-stats From 2a15583cd9186fe63f191e9d30b18cf10a077965 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 01:53:13 +0000 Subject: [PATCH 149/225] t3 --- .github/workflows/packaging_wheels.yml | 4 ++-- pyproject.toml | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 8db4221a..0909cf7d 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,7 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - sccache --show-stats + sccache --show-stats && uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > @@ -156,7 +156,7 @@ jobs: CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_VS_GLOBALS="CLToolExe=sccache.exe;TrackFileAccess=false;UseMultiToolTask=true" SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} PYTHON_GIL=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 diff --git a/pyproject.toml b/pyproject.toml index e2399895..e6ada25b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -434,7 +434,7 @@ before-build = [ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", - "choco uninstall strawberryperl -y --force", + "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", "choco install sccache", "sccache --version" ] @@ -443,6 +443,11 @@ before-test = [ "sccache --show-stats" ] +[[tool.scikit-build.overrides]] +# Use Ninja generator on Windows for sccache compatibility +if.platform-system = "Windows" +cmake.generator = "Ninja" + [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks # for python314.lib From 8ac483a832fe261b3796837367a334a9580a068e Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 01:58:48 +0000 Subject: [PATCH 150/225] install ninja --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e6ada25b..51412604 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -435,8 +435,10 @@ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RES before-build = [ "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", - "choco install sccache", - "sccache --version" + "echo 'Installing sccache and ninja...'", + "choco install sccache ninja", + "sccache --version", + "ninja --version" ] before-test = [ "echo 'Final sccache statistics:'", From 3a5654faeb4d4280f3d4e2cb1a7bb456b8e00b6a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 02:04:56 +0000 Subject: [PATCH 151/225] use ninja --- pyproject.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 51412604..895a116a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -435,10 +435,8 @@ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RES before-build = [ "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", - "echo 'Installing sccache and ninja...'", - "choco install sccache ninja", + "choco install sccache", "sccache --version", - "ninja --version" ] before-test = [ "echo 'Final sccache statistics:'", @@ -447,7 +445,7 @@ before-test = [ [[tool.scikit-build.overrides]] # Use Ninja generator on Windows for sccache compatibility -if.platform-system = "Windows" +if.platform-system = "^win32" cmake.generator = "Ninja" [[tool.cibuildwheel.overrides]] From 56dc713f0c3d118bcbc3805c530b1f3145b45cb9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 02:08:57 +0000 Subject: [PATCH 152/225] force nnnniiinja --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 895a116a..64eae62a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -446,7 +446,7 @@ before-test = [ [[tool.scikit-build.overrides]] # Use Ninja generator on Windows for sccache compatibility if.platform-system = "^win32" -cmake.generator = "Ninja" +cmake.args = ["-G", "Ninja"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From 1d1457110a895a97ec7704ca9f46e9bfc09c54b2 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 02:14:49 +0000 Subject: [PATCH 153/225] double sccache --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 0909cf7d..cd1fe2d6 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -156,7 +156,7 @@ jobs: CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 From d31398672cb4b01cd3879b259878d3ca346b98a2 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 02:35:59 +0000 Subject: [PATCH 154/225] Try mac, let's --- .github/workflows/packaging_wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index cd1fe2d6..f0e2b9d4 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -35,10 +35,10 @@ jobs: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} - - { os: windows-2025, arch: amd64, cibw_system: win } - - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + #- { os: windows-2025, arch: amd64, cibw_system: win } + #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - #- { os: macos-15, arch: arm64, cibw_system: macosx } + - { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } #- { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: @@ -159,7 +159,7 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 7ec8c25b95a0500b54d2f26ff238d719522d7694 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 02:45:02 +0000 Subject: [PATCH 155/225] try to fix windows --- .github/workflows/packaging_wheels.yml | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index f0e2b9d4..a6ad22ba 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -35,10 +35,10 @@ jobs: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} - #- { os: windows-2025, arch: amd64, cibw_system: win } + - { os: windows-2025, arch: amd64, cibw_system: win } #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - - { os: macos-15, arch: arm64, cibw_system: macosx } + #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } #- { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: @@ -159,7 +159,7 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" diff --git a/pyproject.toml b/pyproject.toml index 64eae62a..547a8fe1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -446,7 +446,7 @@ before-test = [ [[tool.scikit-build.overrides]] # Use Ninja generator on Windows for sccache compatibility if.platform-system = "^win32" -cmake.args = ["-G", "Ninja"] +cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From a686e75c99fea2509d8164846930dbaddaaada2f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 02:54:08 +0000 Subject: [PATCH 156/225] win3 --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 547a8fe1..8258e14f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -433,10 +433,14 @@ before-build = [ [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ + "echo 'Setting up Visual Studio environment...'", + "call \"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvars64.bat\"", "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", "choco install sccache", "sccache --version", + "where cl", + "where c++" ] before-test = [ "echo 'Final sccache statistics:'", @@ -446,7 +450,7 @@ before-test = [ [[tool.scikit-build.overrides]] # Use Ninja generator on Windows for sccache compatibility if.platform-system = "^win32" -cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl"] +cmake.args = ["-G", "Ninja"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From 0882af8abb688e3145f16b76b2cc18b18f18a1c9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:01:00 +0000 Subject: [PATCH 157/225] set cl for ninja --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a6ad22ba..9308035b 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -156,10 +156,10 @@ jobs: CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CIBW_BUILD_VERBOSITY: 1 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | echo "Final sccache stats:" From 7215cd231f7212c3afb4b68b33bc4f8f1abc6658 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:08:30 +0000 Subject: [PATCH 158/225] try --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9308035b..86a9b735 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -156,7 +156,7 @@ jobs: CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_ARGS="-DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl" # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" From 0e0a2d67224dfa0e37ccb5d8975027b1a42c5f7f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:12:36 +0000 Subject: [PATCH 159/225] PATH --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 86a9b735..eac9bc58 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -156,7 +156,7 @@ jobs: CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_ARGS="-DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_ARGS="-DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl" PATH="C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Tools\\MSVC\\14.41.34120\\bin\\Hostx64\\x64;C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake\\bin;$PATH" # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" From dc5705e61e82e59610574fe9e7ef337464ac0d07 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:34:38 +0000 Subject: [PATCH 160/225] t6 --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8258e14f..423cc9c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -456,4 +456,8 @@ cmake.args = ["-G", "Ninja"] # Use the correct lib file for 3.14t, without this it looks # for python314.lib select = "cp314t-*" -environment = { CMAKE_ARGS = "-DPython_LIBRARY_RELEASE=python314t.lib" } \ No newline at end of file +environment = { + CMAKE_ARGS = "-DPython_LIBRARY_RELEASE=python314t.lib", + DISTUTILS_USE_SDK = "1", + MSSdk = "1" +} \ No newline at end of file From f6b3152e1b93a7c8bc27fd36597f4b7cb88e05a7 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:38:04 +0000 Subject: [PATCH 161/225] quoting --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 423cc9c9..904c175c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -457,7 +457,7 @@ cmake.args = ["-G", "Ninja"] # for python314.lib select = "cp314t-*" environment = { - CMAKE_ARGS = "-DPython_LIBRARY_RELEASE=python314t.lib", - DISTUTILS_USE_SDK = "1", - MSSdk = "1" + "CMAKE_ARGS" = "-DPython_LIBRARY_RELEASE=python314t.lib", + "DISTUTILS_USE_SDK" = "1", + "MSSdk" = "1" } \ No newline at end of file From 0ad7251e90e66a7182b44f77db4bc8266d5cd05b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:43:30 +0000 Subject: [PATCH 162/225] toml --- pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 904c175c..f0e028a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -456,8 +456,4 @@ cmake.args = ["-G", "Ninja"] # Use the correct lib file for 3.14t, without this it looks # for python314.lib select = "cp314t-*" -environment = { - "CMAKE_ARGS" = "-DPython_LIBRARY_RELEASE=python314t.lib", - "DISTUTILS_USE_SDK" = "1", - "MSSdk" = "1" -} \ No newline at end of file +environment = { "CMAKE_ARGS" = "-DPython_LIBRARY_RELEASE=python314t.lib", "DISTUTILS_USE_SDK" = "1", "MSSdk" = "1" } \ No newline at end of file From b4220ed56616d2d282832a3139ca2d7544bac09e Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:53:15 +0000 Subject: [PATCH 163/225] revert back to half working --- .github/workflows/packaging_wheels.yml | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index eac9bc58..460e34b4 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -128,7 +128,7 @@ jobs: if: ${{ inputs.set-version != '' }} run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] @@ -138,7 +138,7 @@ jobs: - uses: astral-sh/setup-uv@v6 with: version: "0.8.16" - enable-cache: false + enable-cache: true cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} python-version: ${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] @@ -147,16 +147,16 @@ jobs: - name: Install cibuildwheel - run: python -m pip install cibuildwheel==3.1.4 + run: uv pip install cibuildwheel==3.1.4 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - run: python -m cibuildwheel --output-dir wheelhouse + run: uv run python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_ARGS="-DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl" PATH="C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Tools\\MSVC\\14.41.34120\\bin\\Hostx64\\x64;C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake\\bin;$PATH" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" diff --git a/pyproject.toml b/pyproject.toml index f0e028a2..1d840ffd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -456,4 +456,4 @@ cmake.args = ["-G", "Ninja"] # Use the correct lib file for 3.14t, without this it looks # for python314.lib select = "cp314t-*" -environment = { "CMAKE_ARGS" = "-DPython_LIBRARY_RELEASE=python314t.lib", "DISTUTILS_USE_SDK" = "1", "MSSdk" = "1" } \ No newline at end of file +environment = { "CMAKE_ARGS" = "-DPython_LIBRARY_RELEASE=python314t.lib" } \ No newline at end of file From 3283319432bf92a3a8730c3c7cd006a234992ab5 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:57:00 +0000 Subject: [PATCH 164/225] just cache python --- .github/workflows/packaging_wheels.yml | 50 ++++---------------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 460e34b4..14b1e486 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -109,12 +109,12 @@ jobs: clang \ llvm echo "APT package installation completed" - - name: Cache - uses: actions/cache@v4 - with: - path: | - ${{ runner.home }}/.cache/uv - key: build-dir-${{ matrix.python }}-${{ runner.os }}-${{ hashFiles('**/pyproject.toml') }} + # - name: Cache + # uses: actions/cache@v4 + # with: + # path: | + # ${{ runner.home }}/.cache/uv + # key: build-dir-${{ matrix.python }}-${{ runner.os }}-${{ hashFiles('**/pyproject.toml') }} - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -134,40 +134,4 @@ jobs: fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] }} allow-prereleases: true - # Install Astral UV, which will be used as build-frontend for cibuildwheel - - uses: astral-sh/setup-uv@v6 - with: - version: "0.8.16" - enable-cache: true - cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - python-version: ${{ - fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] - }} - - - - - name: Install cibuildwheel - run: uv pip install cibuildwheel==3.1.4 - - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - run: uv run python -m cibuildwheel --output-dir wheelhouse - env: - CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} - CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" - # PYTHON_GIL=1 - # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_BUILD_VERBOSITY: 1 - CIBW_REPAIR_WHEEL_COMMAND_LINUX: | - echo "Final sccache stats:" - sccache --show-stats - auditwheel repair -w {dest_dir} {wheel} - - name: Upload wheel - uses: actions/upload-artifact@v4 - with: - name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - path: wheelhouse/*.whl - compression-level: 0 \ No newline at end of file + \ No newline at end of file From a6826ebe6eccd39c9ae970e9c3e7c857f3e917c3 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 03:58:44 +0000 Subject: [PATCH 165/225] install cibuildwheel --- .github/workflows/packaging_wheels.yml | 51 +++++++------------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 14b1e486..b7c9bf65 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -80,41 +80,6 @@ jobs: script: | core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - name: Install System Dependencies (APT - Ubuntu/Debian) - if: steps.detect-pm.outputs.package_manager == 'apt' - run: | - echo "Installing build dependencies with APT" - sudo apt update - sudo apt-get install -y \ - build-essential \ - cmake \ - git \ - wget \ - curl \ - pkg-config \ - libbz2-dev \ - libffi-dev \ - libgdbm-dev \ - libgdbm-compat-dev \ - liblzma-dev \ - libncurses5-dev \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - lzma \ - lzma-dev \ - tk-dev \ - uuid-dev \ - zlib1g-dev \ - clang \ - llvm - echo "APT package installation completed" - # - name: Cache - # uses: actions/cache@v4 - # with: - # path: | - # ${{ runner.home }}/.cache/uv - # key: build-dir-${{ matrix.python }}-${{ runner.os }}-${{ hashFiles('**/pyproject.toml') }} - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -122,7 +87,6 @@ jobs: cd external/duckdb git fetch origin git checkout ${{ inputs.duckdb-sha }} - # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds - name: Set OVERRIDE_GIT_DESCRIBE if: ${{ inputs.set-version != '' }} @@ -134,4 +98,17 @@ jobs: fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] }} allow-prereleases: true - \ No newline at end of file + # Install Astral UV, which will be used as build-frontend for cibuildwheel + - uses: astral-sh/setup-uv@v6 + with: + version: "0.8.16" + enable-cache: true + cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + python-version: ${{ + fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] + }} + - name: Install cibuildwheel + run: | + uv venv .venv + uv pip install cibuildwheel==3.1.4 + \ No newline at end of file From fd66d3505e692b0d5f2b53f66502547868063a05 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:04:25 +0000 Subject: [PATCH 166/225] the rest --- .github/workflows/packaging_wheels.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b7c9bf65..2b141d09 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -111,4 +111,25 @@ jobs: run: | uv venv .venv uv pip install cibuildwheel==3.1.4 - \ No newline at end of file + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels + run: uv run python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" + # PYTHON_GIL=1 + # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_BUILD_VERBOSITY: 3 + CIBW_REPAIR_WHEEL_COMMAND_LINUX: | + echo "Final sccache stats:" + sccache --show-stats + auditwheel repair -w {dest_dir} {wheel} + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + path: wheelhouse/*.whl + compression-level: 0 \ No newline at end of file From b55429b37ceaf248fd408700d47330c23d84cd71 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:08:25 +0000 Subject: [PATCH 167/225] s3 --- .github/workflows/packaging_wheels.yml | 32 +++----------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 2b141d09..55a77366 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -91,13 +91,6 @@ jobs: - name: Set OVERRIDE_GIT_DESCRIBE if: ${{ inputs.set-version != '' }} run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v6 - with: - python-version: ${{ - fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] - }} - allow-prereleases: true # Install Astral UV, which will be used as build-frontend for cibuildwheel - uses: astral-sh/setup-uv@v6 with: @@ -109,27 +102,8 @@ jobs: }} - name: Install cibuildwheel run: | + uv python install + echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" uv venv .venv uv pip install cibuildwheel==3.1.4 - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - run: uv run python -m cibuildwheel --output-dir wheelhouse - env: - CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} - CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" - # PYTHON_GIL=1 - # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_BUILD_VERBOSITY: 3 - CIBW_REPAIR_WHEEL_COMMAND_LINUX: | - echo "Final sccache stats:" - sccache --show-stats - auditwheel repair -w {dest_dir} {wheel} - - name: Upload wheel - uses: actions/upload-artifact@v4 - with: - name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - path: wheelhouse/*.whl - compression-level: 0 \ No newline at end of file + \ No newline at end of file From 09faf75c078cc78c4d01fec605f1413e59d479e9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:10:39 +0000 Subject: [PATCH 168/225] b --- .github/workflows/packaging_wheels.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 55a77366..32d320cc 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -106,4 +106,25 @@ jobs: echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" uv venv .venv uv pip install cibuildwheel==3.1.4 - \ No newline at end of file + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels + run: uv run python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" + # PYTHON_GIL=1 + # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_BUILD_VERBOSITY: 3 + CIBW_REPAIR_WHEEL_COMMAND_LINUX: | + echo "Final sccache stats:" + sccache --show-stats + auditwheel repair -w {dest_dir} {wheel} + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + path: wheelhouse/*.whl + compression-level: 0 \ No newline at end of file From e3d50ced0cd9ba957b20ba7220e0de84cb513f1a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:14:18 +0000 Subject: [PATCH 169/225] pin setup-uv version --- .github/workflows/packaging_wheels.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 32d320cc..31ed6920 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -96,13 +96,11 @@ jobs: with: version: "0.8.16" enable-cache: true - cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - python-version: ${{ - fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] - }} + cache-suffix: -3.13-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + python-version: 3.13 # this is only for the host, cibuildwheel will install the target python versions in the container - name: Install cibuildwheel run: | - uv python install + uv python install echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" uv venv .venv uv pip install cibuildwheel==3.1.4 From 3760692aa71aba8d6925ddd002b3ecab5fe53022 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:19:18 +0000 Subject: [PATCH 170/225] 3.11 --- .github/workflows/packaging_wheels.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 31ed6920..f0cb32bb 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -96,13 +96,12 @@ jobs: with: version: "0.8.16" enable-cache: true + activate-environment: true cache-suffix: -3.13-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - python-version: 3.13 # this is only for the host, cibuildwheel will install the target python versions in the container + python-version: 3.11 # this is only for the host, cibuildwheel will install the target python versions in the container - name: Install cibuildwheel run: | - uv python install echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" - uv venv .venv uv pip install cibuildwheel==3.1.4 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels run: uv run python -m cibuildwheel --output-dir wheelhouse From 6c6bbeadd52bdf603177a190269cdb537cb49d77 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:24:58 +0000 Subject: [PATCH 171/225] no-sync --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index f0cb32bb..9e0aec50 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -97,14 +97,14 @@ jobs: version: "0.8.16" enable-cache: true activate-environment: true - cache-suffix: -3.13-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + cache-suffix: -3.11-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} python-version: 3.11 # this is only for the host, cibuildwheel will install the target python versions in the container - name: Install cibuildwheel run: | echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" uv pip install cibuildwheel==3.1.4 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - run: uv run python -m cibuildwheel --output-dir wheelhouse + run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} From bacf97b74c0d23e7b36348bd8934bd3ffdb7ec8c Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:30:26 +0000 Subject: [PATCH 172/225] version --- .github/workflows/packaging_wheels.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9e0aec50..6ec66c80 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -97,8 +97,10 @@ jobs: version: "0.8.16" enable-cache: true activate-environment: true - cache-suffix: -3.11-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - python-version: 3.11 # this is only for the host, cibuildwheel will install the target python versions in the container + cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + python-version: ${{ + fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] + }} - name: Install cibuildwheel run: | echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" From 24f445d56effe7da804e3ddaf2e23c2636bccce2 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:50:34 +0000 Subject: [PATCH 173/225] zz --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 6ec66c80..a2903bbc 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -112,7 +112,7 @@ jobs: CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_C_COMPILER="cl" CMAKE_CXX_COMPILER="cl" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON # PYTHON_GIL=1 # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" diff --git a/pyproject.toml b/pyproject.toml index 1d840ffd..8d4b3132 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -448,9 +448,10 @@ before-test = [ ] [[tool.scikit-build.overrides]] -# Use Ninja generator on Windows for sccache compatibility +# Use Ninja generator on Windows for sccache compatibility +# Force MSVC compiler and enable sccache launcher if.platform-system = "^win32" -cmake.args = ["-G", "Ninja"] +cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From af42e970f77e1cf583d2838fbcca18ee46818f19 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:54:19 +0000 Subject: [PATCH 174/225] zz --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8d4b3132..6b8d3eab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -451,7 +451,7 @@ before-test = [ # Use Ninja generator on Windows for sccache compatibility # Force MSVC compiler and enable sccache launcher if.platform-system = "^win32" -cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] +cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_CXX_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From 105d6e93fea0e894d31050f15468cffc73f70f4b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 04:58:41 +0000 Subject: [PATCH 175/225] paths --- pyproject.toml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6b8d3eab..47ef3e37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -439,8 +439,14 @@ before-build = [ "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", "choco install sccache", "sccache --version", + "echo 'Checking for build tools...'", "where cl", - "where c++" + "where link", + "where rc", + "where mt", + "echo 'Environment variables:'", + "echo 'VCINSTALLDIR=%VCINSTALLDIR%'", + "echo 'WindowsSdkDir=%WindowsSdkDir%'" ] before-test = [ "echo 'Final sccache statistics:'", @@ -451,7 +457,7 @@ before-test = [ # Use Ninja generator on Windows for sccache compatibility # Force MSVC compiler and enable sccache launcher if.platform-system = "^win32" -cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_CXX_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] +cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_CXX_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_RC_COMPILER=C:/Program Files (x86)/Windows Kits/10/bin/10.0.26100.0/x64/rc.exe", "-DCMAKE_MT=C:/Program Files (x86)/Windows Kits/10/bin/10.0.26100.0/x64/mt.exe", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From 20bb15dbb29a871378567be2cbedb74fe9b9478b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:04:18 +0000 Subject: [PATCH 176/225] try again --- pyproject.toml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 47ef3e37..0581a91e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -433,8 +433,9 @@ before-build = [ [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ - "echo 'Setting up Visual Studio environment...'", - "call \"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvars64.bat\"", + "echo 'Setting up Visual Studio environment with pinned Windows SDK...'", + "call \"%ProgramFiles(x86)%\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\Tools\\VsDevCmd.bat\" -winsdk=10.0.26100.0", + "set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0", "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", "choco install sccache", @@ -446,7 +447,9 @@ before-build = [ "where mt", "echo 'Environment variables:'", "echo 'VCINSTALLDIR=%VCINSTALLDIR%'", - "echo 'WindowsSdkDir=%WindowsSdkDir%'" + "echo 'WindowsSdkDir=%WindowsSdkDir%'", + "echo 'WindowsSDKVersion=%WindowsSDKVersion%'", + "echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%'" ] before-test = [ "echo 'Final sccache statistics:'", @@ -457,7 +460,7 @@ before-test = [ # Use Ninja generator on Windows for sccache compatibility # Force MSVC compiler and enable sccache launcher if.platform-system = "^win32" -cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_CXX_COMPILER=C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe", "-DCMAKE_RC_COMPILER=C:/Program Files (x86)/Windows Kits/10/bin/10.0.26100.0/x64/rc.exe", "-DCMAKE_MT=C:/Program Files (x86)/Windows Kits/10/bin/10.0.26100.0/x64/mt.exe", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] +cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From b11283fa5de80a11f3eee9535318a0c71db17f2f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:09:44 +0000 Subject: [PATCH 177/225] dynamic --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0581a91e..85712a63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -434,7 +434,7 @@ before-build = [ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ "echo 'Setting up Visual Studio environment with pinned Windows SDK...'", - "call \"%ProgramFiles(x86)%\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\Tools\\VsDevCmd.bat\" -winsdk=10.0.26100.0", + "for /f \"usebackq tokens=*\" %i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do (call \"%i\\Common7\\Tools\\VsDevCmd.bat\" -winsdk=10.0.26100.0)", "set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0", "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", From 7f0291592735f7182ffad0ea30cb5feead4c5726 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:14:15 +0000 Subject: [PATCH 178/225] z --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 85712a63..5ad468f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -442,9 +442,11 @@ before-build = [ "sccache --version", "echo 'Checking for build tools...'", "where cl", + "where c++", "where link", "where rc", "where mt", + "echo 'PATH=%PATH%'", "echo 'Environment variables:'", "echo 'VCINSTALLDIR=%VCINSTALLDIR%'", "echo 'WindowsSdkDir=%WindowsSdkDir%'", @@ -460,7 +462,7 @@ before-test = [ # Use Ninja generator on Windows for sccache compatibility # Force MSVC compiler and enable sccache launcher if.platform-system = "^win32" -cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] +cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From 05017caa8d09486002bb3e0d88f6c8d55a7de4f1 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:23:12 +0000 Subject: [PATCH 179/225] c --- pyproject.toml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5ad468f0..4ecf5d9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -433,25 +433,7 @@ before-build = [ [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ - "echo 'Setting up Visual Studio environment with pinned Windows SDK...'", - "for /f \"usebackq tokens=*\" %i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do (call \"%i\\Common7\\Tools\\VsDevCmd.bat\" -winsdk=10.0.26100.0)", - "set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0", - "echo 'Removing Strawberry Perl ccache to avoid conflicts...'", - "del \"C:\\Strawberry\\c\\bin\\ccache.exe\" ", - "choco install sccache", - "sccache --version", - "echo 'Checking for build tools...'", - "where cl", - "where c++", - "where link", - "where rc", - "where mt", - "echo 'PATH=%PATH%'", - "echo 'Environment variables:'", - "echo 'VCINSTALLDIR=%VCINSTALLDIR%'", - "echo 'WindowsSdkDir=%WindowsSdkDir%'", - "echo 'WindowsSDKVersion=%WindowsSDKVersion%'", - "echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%'" + "echo 'Setting up Visual Studio environment with pinned Windows SDK...' && for /f \"usebackq tokens=*\" %%i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call \"%%i\\Common7\\Tools\\VsDevCmd.bat\" -arch=x64 -winsdk=10.0.26100.0 && echo 'After VsDevCmd.bat - checking if cl is now available:' && where cl && set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0 && echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%' && del \"C:\\Strawberry\\c\\bin\\ccache.exe\" && choco install sccache && sccache --version && echo 'Final check - cl should be available:' && where cl" ] before-test = [ "echo 'Final sccache statistics:'", From af833982c777bdb4648ab7a2f18b91e30b8b66c8 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:26:06 +0000 Subject: [PATCH 180/225] fix escapt --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4ecf5d9b..2f2cee41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -433,7 +433,7 @@ before-build = [ [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ - "echo 'Setting up Visual Studio environment with pinned Windows SDK...' && for /f \"usebackq tokens=*\" %%i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call \"%%i\\Common7\\Tools\\VsDevCmd.bat\" -arch=x64 -winsdk=10.0.26100.0 && echo 'After VsDevCmd.bat - checking if cl is now available:' && where cl && set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0 && echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%' && del \"C:\\Strawberry\\c\\bin\\ccache.exe\" && choco install sccache && sccache --version && echo 'Final check - cl should be available:' && where cl" + "echo 'Setting up Visual Studio environment with pinned Windows SDK...' && for /f \"usebackq tokens=*\" %i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call \"%i\\Common7\\Tools\\VsDevCmd.bat\" -arch=x64 -winsdk=10.0.26100.0 && echo 'After VsDevCmd.bat - checking if cl is now available:' && where cl && set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0 && echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%' && del \"C:\\Strawberry\\c\\bin\\ccache.exe\" && choco install sccache && sccache --version && echo 'Final check - cl should be available:' && where cl" ] before-test = [ "echo 'Final sccache statistics:'", From 02533157519237cd05f2a941103cbcc2ec4ec77b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:27:38 +0000 Subject: [PATCH 181/225] d --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2f2cee41..86e22142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -433,7 +433,7 @@ before-build = [ [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ - "echo 'Setting up Visual Studio environment with pinned Windows SDK...' && for /f \"usebackq tokens=*\" %i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call \"%i\\Common7\\Tools\\VsDevCmd.bat\" -arch=x64 -winsdk=10.0.26100.0 && echo 'After VsDevCmd.bat - checking if cl is now available:' && where cl && set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0 && echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%' && del \"C:\\Strawberry\\c\\bin\\ccache.exe\" && choco install sccache && sccache --version && echo 'Final check - cl should be available:' && where cl" + "echo 'Setting up Visual Studio environment with pinned Windows SDK...' && for /f \"usebackq tokens=*\" %i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call \"%i\\Common7\\Tools\\VsDevCmd.bat\" && echo 'After VsDevCmd.bat - checking if cl is now available:' && where cl && set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0 && echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%' && del \"C:\\Strawberry\\c\\bin\\ccache.exe\" && choco install sccache && sccache --version && echo 'Final check - cl should be available:' && where cl" ] before-test = [ "echo 'Final sccache statistics:'", From 639f331604d54e6b9b4ab7760b2bafe50218e8a0 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:38:12 +0000 Subject: [PATCH 182/225] move to action --- .github/workflows/packaging_wheels.yml | 7 +++++++ pyproject.toml | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a2903bbc..8f28e57a 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -105,6 +105,13 @@ jobs: run: | echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" uv pip install cibuildwheel==3.1.4 + - name: Set up Visual Studio environment (Windows) + if: matrix.platform.cibw_system == 'win' + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call "%%i\Common7\Tools\VsDevCmd.bat" + echo CC=cl >> %GITHUB_ENV% + echo CXX=cl >> %GITHUB_ENV% - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse env: diff --git a/pyproject.toml b/pyproject.toml index 86e22142..94b06716 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -431,9 +431,12 @@ before-build = [ ] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL", "CC", "CXX"] before-build = [ - "echo 'Setting up Visual Studio environment with pinned Windows SDK...' && for /f \"usebackq tokens=*\" %i in (`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call \"%i\\Common7\\Tools\\VsDevCmd.bat\" && echo 'After VsDevCmd.bat - checking if cl is now available:' && where cl && set CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=10.0.26100.0 && echo 'CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION=%CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION%' && del \"C:\\Strawberry\\c\\bin\\ccache.exe\" && choco install sccache && sccache --version && echo 'Final check - cl should be available:' && where cl" + "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"", + "choco install sccache", + "sccache --version", + "echo 'Verifying cl is available:' && where cl" ] before-test = [ "echo 'Final sccache statistics:'", From daa0e76bc79eca18ed0ad9a5fd22bfe3c1172e04 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:43:47 +0000 Subject: [PATCH 183/225] sd --- .github/workflows/packaging_wheels.yml | 9 ++++++--- pyproject.toml | 3 +-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 8f28e57a..3c0851c2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -109,9 +109,12 @@ jobs: if: matrix.platform.cibw_system == 'win' shell: cmd run: | - for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do call "%%i\Common7\Tools\VsDevCmd.bat" - echo CC=cl >> %GITHUB_ENV% - echo CXX=cl >> %GITHUB_ENV% + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set VSINSTALLDIR=%%i + for /f "usebackq tokens=*" %%i in (`dir /b "%VSINSTALLDIR%\VC\Tools\MSVC"`) do set MSVCVER=%%i + set CLPATH=%VSINSTALLDIR%\VC\Tools\MSVC\%MSVCVER%\bin\Hostx64\x64\cl.exe + echo CC=%CLPATH% >> %GITHUB_ENV% + echo CXX=%CLPATH% >> %GITHUB_ENV% + echo Compiler paths set: %CLPATH% - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse env: diff --git a/pyproject.toml b/pyproject.toml index 94b06716..4658a647 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -435,8 +435,7 @@ environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RES before-build = [ "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"", "choco install sccache", - "sccache --version", - "echo 'Verifying cl is available:' && where cl" + "sccache --version" ] before-test = [ "echo 'Final sccache statistics:'", From 1afe60e6d2e69f35db7eee184cae1bb5cbfd68a2 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:52:14 +0000 Subject: [PATCH 184/225] t --- .github/workflows/packaging_wheels.yml | 21 ++++++++++----------- pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 3c0851c2..e0170375 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -105,26 +105,25 @@ jobs: run: | echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" uv pip install cibuildwheel==3.1.4 - - name: Set up Visual Studio environment (Windows) + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Windows) if: matrix.platform.cibw_system == 'win' shell: cmd run: | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set VSINSTALLDIR=%%i - for /f "usebackq tokens=*" %%i in (`dir /b "%VSINSTALLDIR%\VC\Tools\MSVC"`) do set MSVCVER=%%i - set CLPATH=%VSINSTALLDIR%\VC\Tools\MSVC\%MSVCVER%\bin\Hostx64\x64\cl.exe - echo CC=%CLPATH% >> %GITHUB_ENV% - echo CXX=%CLPATH% >> %GITHUB_ENV% - echo Compiler paths set: %CLPATH% - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels + call "%VSINSTALLDIR%\Common7\Tools\VsDevCmd.bat" + uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON + CIBW_BUILD_VERBOSITY: 3 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) + if: matrix.platform.cibw_system != 'win' run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - # SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1 CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON - # PYTHON_GIL=1 - # SCCACHE_GHA_ENABLED=on ACTIONS_RUNTIME_TOKEN=${{ github.token }} ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }} ACTIONS_CACHE_SERVICE_V2=1 SCCACHE_C_CUSTOM_CACHE_BUSTER=${{ matrix.python }}-${{ matrix.platform.arch }} # CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CIBW_BUILD_VERBOSITY: 3 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | diff --git a/pyproject.toml b/pyproject.toml index 4658a647..0369fea5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -431,7 +431,7 @@ before-build = [ ] [tool.cibuildwheel.windows] -environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL", "CC", "CXX"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] before-build = [ "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"", "choco install sccache", From 021f6f2ca8d274d85006274970c50fe0bb97e198 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 05:59:18 +0000 Subject: [PATCH 185/225] deb --- .github/workflows/packaging_wheels.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index e0170375..e0cc196c 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -111,6 +111,14 @@ jobs: run: | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set VSINSTALLDIR=%%i call "%VSINSTALLDIR%\Common7\Tools\VsDevCmd.bat" + echo "After VsDevCmd - checking PATH contains Visual Studio:" + echo %PATH% | findstr /i "Microsoft Visual Studio" + echo "Checking if cl is available:" + where cl + echo "Environment variables that should be set:" + echo CC=%CC% + echo CXX=%CXX% + echo LIB=%LIB% uv run --no-sync python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} From 435e5794fdbd07d666f944c78b61e81a466c7eed Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:00:26 +0000 Subject: [PATCH 186/225] r --- .github/workflows/packaging_wheels.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index e0cc196c..5357acca 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -109,17 +109,14 @@ jobs: if: matrix.platform.cibw_system == 'win' shell: cmd run: | - for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set VSINSTALLDIR=%%i - call "%VSINSTALLDIR%\Common7\Tools\VsDevCmd.bat" - echo "After VsDevCmd - checking PATH contains Visual Studio:" - echo %PATH% | findstr /i "Microsoft Visual Studio" - echo "Checking if cl is available:" - where cl - echo "Environment variables that should be set:" - echo CC=%CC% - echo CXX=%CXX% - echo LIB=%LIB% - uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + call "%%i\Common7\Tools\VsDevCmd.bat" + echo "After VsDevCmd - checking PATH contains Visual Studio:" + echo !PATH! | findstr /i "Microsoft Visual Studio" + echo "Checking if cl is available:" + where cl + uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + ) env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} From c5649b17ba0ca6a875f0050d50127aa6b213e49a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:04:42 +0000 Subject: [PATCH 187/225] z --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0369fea5..5e923ffe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -444,9 +444,9 @@ before-test = [ [[tool.scikit-build.overrides]] # Use Ninja generator on Windows for sccache compatibility -# Force MSVC compiler and enable sccache launcher +# Let CMake find the compiler and use sccache launcher if.platform-system = "^win32" -cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] +cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From f4bae55b7d4887a6f61837c8127fa742225681fa Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:05:40 +0000 Subject: [PATCH 188/225] x --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5357acca..a6e3c33a 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -120,7 +120,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CIBW_BUILD_VERBOSITY: 3 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) if: matrix.platform.cibw_system != 'win' diff --git a/pyproject.toml b/pyproject.toml index 5e923ffe..b52b9578 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -443,10 +443,9 @@ before-test = [ ] [[tool.scikit-build.overrides]] -# Use Ninja generator on Windows for sccache compatibility -# Let CMake find the compiler and use sccache launcher +# Use Ninja generator on Windows for faster builds if.platform-system = "^win32" -cmake.args = ["-G", "Ninja", "-DCMAKE_C_COMPILER_LAUNCHER=sccache", "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"] +cmake.args = ["-G", "Ninja"] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From ed392e0d532b856d4215dd15321d5fbc518a908c Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:09:34 +0000 Subject: [PATCH 189/225] f --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a6e3c33a..9a1d6b5c 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -120,7 +120,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_CXX_FLAGS="/FS" CMAKE_C_FLAGS="/FS" CIBW_BUILD_VERBOSITY: 3 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) if: matrix.platform.cibw_system != 'win' From f6931a6fe8da80c113647ce6c373ad8df69a6ea7 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:16:38 +0000 Subject: [PATCH 190/225] sd --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9a1d6b5c..1218fb17 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -120,7 +120,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_CXX_FLAGS="/FS" CMAKE_C_FLAGS="/FS" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded CMAKE_C_FLAGS_RELWITHDEBINFO="/Z7" CMAKE_CXX_FLAGS_RELWITHDEBINFO="/Z7" CIBW_BUILD_VERBOSITY: 3 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) if: matrix.platform.cibw_system != 'win' From db880f07663dda34b45744539d86d22b6178e26b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:22:05 +0000 Subject: [PATCH 191/225] ad --- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 1218fb17..5357acca 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -120,7 +120,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded CMAKE_C_FLAGS_RELWITHDEBINFO="/Z7" CMAKE_CXX_FLAGS_RELWITHDEBINFO="/Z7" + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON CIBW_BUILD_VERBOSITY: 3 - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) if: matrix.platform.cibw_system != 'win' diff --git a/pyproject.toml b/pyproject.toml index b52b9578..671dd5f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -443,9 +443,17 @@ before-test = [ ] [[tool.scikit-build.overrides]] -# Use Ninja generator on Windows for faster builds +# Use Ninja generator on Windows with embedded debug info for sccache compatibility if.platform-system = "^win32" -cmake.args = ["-G", "Ninja"] +cmake.args = [ + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_C_COMPILER_LAUNCHER=sccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache" +] [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t, without this it looks From 1f33aa7bb00db7dab81770b92e458c7724bf2695 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:25:53 +0000 Subject: [PATCH 192/225] asd --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 671dd5f0..33a4056d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -450,9 +450,7 @@ cmake.args = [ "-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", - "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", - "-DCMAKE_C_COMPILER_LAUNCHER=sccache", - "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache" + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG" ] [[tool.cibuildwheel.overrides]] From 30fefd3211eb67fbe92122862bb5f7012fb96f1c Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:30:06 +0000 Subject: [PATCH 193/225] blank --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 33a4056d..7ffbba7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -450,7 +450,9 @@ cmake.args = [ "-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", - "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG" + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_C_COMPILER_LAUNCHER=", + "-DCMAKE_CXX_COMPILER_LAUNCHER=" ] [[tool.cibuildwheel.overrides]] From 8ec7940e0de75542c3d0d337ca52908edc51ab18 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 06:35:38 +0000 Subject: [PATCH 194/225] all --- .github/workflows/packaging_wheels.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5357acca..e5e3862a 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,13 +34,13 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: - #- { os: self-hosted, arch: x86_64, cibw_system: manylinux} + - { os: self-hosted, arch: x86_64, cibw_system: manylinux} - { os: windows-2025, arch: amd64, cibw_system: win } - #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - #- { os: macos-15, arch: arm64, cibw_system: macosx } - #- { os: macos-15, arch: universal2, cibw_system: macosx } - #- { os: macos-13, arch: x86_64, cibw_system: macosx } + - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + - { os: macos-15, arch: arm64, cibw_system: macosx } + - { os: macos-15, arch: universal2, cibw_system: macosx } + - { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: From 8d2eb55010fb39753940c62fe59a46f65353d360 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 11:59:44 +0000 Subject: [PATCH 195/225] win.. land the plane --- .github/workflows/packaging_wheels.yml | 11 +++++------ pyproject.toml | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index e5e3862a..65e60160 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,13 +34,12 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: - - { os: self-hosted, arch: x86_64, cibw_system: manylinux} - { os: windows-2025, arch: amd64, cibw_system: win } - - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - - { os: macos-15, arch: arm64, cibw_system: macosx } - - { os: macos-15, arch: universal2, cibw_system: macosx } - - { os: macos-13, arch: x86_64, cibw_system: macosx } + #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + #- { os: macos-15, arch: arm64, cibw_system: macosx } + #- { os: macos-15, arch: universal2, cibw_system: macosx } + #- { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: diff --git a/pyproject.toml b/pyproject.toml index 7ffbba7a..71e4f2b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -456,7 +456,6 @@ cmake.args = [ ] [[tool.cibuildwheel.overrides]] -# Use the correct lib file for 3.14t, without this it looks -# for python314.lib +# Use the correct lib file for 3.14t free-threading build select = "cp314t-*" -environment = { "CMAKE_ARGS" = "-DPython_LIBRARY_RELEASE=python314t.lib" } \ No newline at end of file +environment = { "CMAKE_ARGS" = "-DPython3_FIND_ABI=ANY;ANY;ANY;ON -DPython3_LIBRARY=python314t.lib" } \ No newline at end of file From 5dbc94aba3200923d33b022036712f515a875f89 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 12:18:07 +0000 Subject: [PATCH 196/225] try again --- .github/workflows/packaging_wheels.yml | 14 ++++++-------- pyproject.toml | 4 +++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 65e60160..0c3ea173 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -52,15 +52,14 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - sccache --show-stats && uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} - ACTIONS_CACHE_SERVICE_V2: "1" - PYTHON_GIL: "1" + ACTIONS_CACHE_SERVICE_V2: "1" + PYTHON_GIL: "1" # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions. steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 @@ -110,10 +109,10 @@ jobs: run: | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( call "%%i\Common7\Tools\VsDevCmd.bat" - echo "After VsDevCmd - checking PATH contains Visual Studio:" - echo !PATH! | findstr /i "Microsoft Visual Studio" - echo "Checking if cl is available:" - where cl + # echo "After VsDevCmd - checking PATH contains Visual Studio:" + # echo !PATH! | findstr /i "Microsoft Visual Studio" + # echo "Checking if cl is available:" + # where cl uv run --no-sync python -m cibuildwheel --output-dir wheelhouse ) env: @@ -131,7 +130,6 @@ jobs: CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CIBW_BUILD_VERBOSITY: 3 CIBW_REPAIR_WHEEL_COMMAND_LINUX: | - echo "Final sccache stats:" sccache --show-stats auditwheel repair -w {dest_dir} {wheel} - name: Upload wheel diff --git a/pyproject.toml b/pyproject.toml index 71e4f2b0..3dcb38df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -452,7 +452,9 @@ cmake.args = [ "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", "-DCMAKE_C_COMPILER_LAUNCHER=", - "-DCMAKE_CXX_COMPILER_LAUNCHER=" + "-DCMAKE_CXX_COMPILER_LAUNCHER=", + "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", + "-DPython3_LIBRARY=python314t.lib" ] [[tool.cibuildwheel.overrides]] From ba735b6a71d067fec93bec6d9f4f90799dee6947 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 12:21:55 +0000 Subject: [PATCH 197/225] comments --- .github/workflows/packaging_wheels.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 0c3ea173..20eb9ecd 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -59,7 +59,8 @@ jobs: SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" - PYTHON_GIL: "1" # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions. + PYTHON_GIL: "1" + # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions. steps: - name: Checkout DuckDB Python uses: actions/checkout@v5 @@ -109,10 +110,6 @@ jobs: run: | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( call "%%i\Common7\Tools\VsDevCmd.bat" - # echo "After VsDevCmd - checking PATH contains Visual Studio:" - # echo !PATH! | findstr /i "Microsoft Visual Studio" - # echo "Checking if cl is available:" - # where cl uv run --no-sync python -m cibuildwheel --output-dir wheelhouse ) env: From 075ada9a4d2028d649e76c6e62e6becd7e73055f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 12:31:07 +0000 Subject: [PATCH 198/225] t --- pyproject.toml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3dcb38df..592c9eaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -432,18 +432,24 @@ before-build = [ [tool.cibuildwheel.windows] environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +# if ccache is in the path, it's chosen before sccache +# and, uninstalling strawberry is much slower than just deleting the entry before-build = [ "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"", "choco install sccache", "sccache --version" ] before-test = [ - "echo 'Final sccache statistics:'", "sccache --show-stats" ] [[tool.scikit-build.overrides]] -# Use Ninja generator on Windows with embedded debug info for sccache compatibility +# To get sccache to work, needed a few things +# Ninja +# /Z7 so we don't write parallel PDBs +# The empty COMPILER_LAUNCHERS are because otherwise sccache is called twice (sccache sccache cl.exe ...) +# The LIBRARY is because it picks up python314.lib, instead of python314t.lib +# TODO: Dynamically add the "t" for future versions (ie: python315t) if.platform-system = "^win32" cmake.args = [ "-G", "Ninja", @@ -454,7 +460,9 @@ cmake.args = [ "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", - "-DPython3_LIBRARY=python314t.lib" + "-DPython3_LIBRARY_RELEASE=python314t.lib", + "-DPython_LIBRARY=python314t.lib", + "-DPython_LIBRARY_RELEASE=python314t.lib" ] [[tool.cibuildwheel.overrides]] From 573a83a8950f2bd2e6cfb991edf7a497a77f56cc Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 12:43:43 +0000 Subject: [PATCH 199/225] more debug --- pyproject.toml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 592c9eaa..6c438e04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -459,10 +459,16 @@ cmake.args = [ "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", + "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", "-DPython3_LIBRARY_RELEASE=python314t.lib", "-DPython_LIBRARY=python314t.lib", - "-DPython_LIBRARY_RELEASE=python314t.lib" + "-DPython_LIBRARY_RELEASE=python314t.lib", + "--debug-output", + "--debug-find", + "--debug-find-pkg=Python3", + "--trace-expand", + "--log-level=DEBUG" ] [[tool.cibuildwheel.overrides]] From b9fae5cd95fe36b28091a78a56717317b6f5bafb Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 12:48:09 +0000 Subject: [PATCH 200/225] trying to get uv to cache --- .github/workflows/packaging_wheels.yml | 32 +------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 20eb9ecd..9a47af51 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -104,34 +104,4 @@ jobs: run: | echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" uv pip install cibuildwheel==3.1.4 - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Windows) - if: matrix.platform.cibw_system == 'win' - shell: cmd - run: | - for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( - call "%%i\Common7\Tools\VsDevCmd.bat" - uv run --no-sync python -m cibuildwheel --output-dir wheelhouse - ) - env: - CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} - CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON - CIBW_BUILD_VERBOSITY: 3 - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) - if: matrix.platform.cibw_system != 'win' - run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse - env: - CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} - CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" - CIBW_BUILD_VERBOSITY: 3 - CIBW_REPAIR_WHEEL_COMMAND_LINUX: | - sccache --show-stats - auditwheel repair -w {dest_dir} {wheel} - - name: Upload wheel - uses: actions/upload-artifact@v4 - with: - name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - path: wheelhouse/*.whl - compression-level: 0 \ No newline at end of file + \ No newline at end of file From 4abfeceb9f17d7cfa90c0266eb591d86d5164e5a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 12:50:37 +0000 Subject: [PATCH 201/225] tak 2 --- .github/workflows/packaging_wheels.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 9a47af51..7aac8a41 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -99,9 +99,4 @@ jobs: cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} python-version: ${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] - }} - - name: Install cibuildwheel - run: | - echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" - uv pip install cibuildwheel==3.1.4 - \ No newline at end of file + }} \ No newline at end of file From 6fa6786f3a64997db23e2076b26c70e814a2ff38 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:01:44 +0000 Subject: [PATCH 202/225] -b --- .github/workflows/packaging_wheels.yml | 39 ++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 7aac8a41..132f14eb 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -57,7 +57,7 @@ jobs: CIBW_TEST_COMMAND: > uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" - SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }}-b ACTIONS_CACHE_SERVICE_V2: "1" PYTHON_GIL: "1" # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions. @@ -99,4 +99,39 @@ jobs: cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} python-version: ${{ fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] - }} \ No newline at end of file + }} + - name: Install cibuildwheel + run: | + echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" + uv pip install cibuildwheel==3.1.4 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Windows) + if: matrix.platform.cibw_system == 'win' + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + call "%%i\Common7\Tools\VsDevCmd.bat" + uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + ) + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON + CIBW_BUILD_VERBOSITY: 3 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) + if: matrix.platform.cibw_system != 'win' + run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_BUILD_VERBOSITY: 3 + CIBW_REPAIR_WHEEL_COMMAND_LINUX: | + sccache --show-stats + auditwheel repair -w {dest_dir} {wheel} + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + path: wheelhouse/*.whl + compression-level: 0 \ No newline at end of file From 3bd0a611537e9581a7fc65eb327325da6cf0d0e9 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:15:14 +0000 Subject: [PATCH 203/225] no auto --- .github/workflows/packaging_wheels.yml | 3 ++- pyproject.toml | 21 ++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 132f14eb..d1723943 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -57,7 +57,7 @@ jobs: CIBW_TEST_COMMAND: > uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" - SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }}-b + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" PYTHON_GIL: "1" # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions. @@ -109,6 +109,7 @@ jobs: shell: cmd run: | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + set VSCMD_ARG_TGT_ARCH=x64 call "%%i\Common7\Tools\VsDevCmd.bat" uv run --no-sync python -m cibuildwheel --output-dir wheelhouse ) diff --git a/pyproject.toml b/pyproject.toml index 6c438e04..8fcf07ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -455,21 +455,20 @@ cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", - "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", - "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPYTHON_NO_LINK", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPYTHON_NO_LINK", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", + "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", "-DPython3_FIND_STRATEGY=LOCATION", - "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", - "-DPython3_LIBRARY_RELEASE=python314t.lib", - "-DPython_LIBRARY=python314t.lib", - "-DPython_LIBRARY_RELEASE=python314t.lib", - "--debug-output", - "--debug-find", - "--debug-find-pkg=Python3", - "--trace-expand", - "--log-level=DEBUG" + "-DPython3_FIND_ABI=ANY;ANY;ANY;ON" ] + # "--debug-output", + # "--debug-find", + # "--debug-find-pkg=Python3", + # "--trace-expand", + # "--log-level=DEBUG" [[tool.cibuildwheel.overrides]] # Use the correct lib file for 3.14t free-threading build From 532e751850b65370f1b0a4d38a8fd99138299a7d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:33:56 +0000 Subject: [PATCH 204/225] set arch --- pyproject.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8fcf07ed..2e445050 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -462,7 +462,9 @@ cmake.args = [ "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", "-DPython3_FIND_STRATEGY=LOCATION", - "-DPython3_FIND_ABI=ANY;ANY;ANY;ON" + "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", + "-DPython3_LIBRARY=python314t.lib", + "-DCMAKE_GENERATOR_PLATFORM=x64" ] # "--debug-output", # "--debug-find", @@ -470,7 +472,3 @@ cmake.args = [ # "--trace-expand", # "--log-level=DEBUG" -[[tool.cibuildwheel.overrides]] -# Use the correct lib file for 3.14t free-threading build -select = "cp314t-*" -environment = { "CMAKE_ARGS" = "-DPython3_FIND_ABI=ANY;ANY;ANY;ON -DPython3_LIBRARY=python314t.lib" } \ No newline at end of file From 775a0f705ee0813b28f177d9790754bd3972bbcb Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:34:36 +0000 Subject: [PATCH 205/225] machine --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2e445050..b69214a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -459,8 +459,8 @@ cmake.args = [ "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPYTHON_NO_LINK", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", - "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", - "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /MACHINE:X64", + "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /MACHINE:X64", "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", "-DPython3_LIBRARY=python314t.lib", From 6716110c13e986f6c56ae8b061c8b8ee11f30bb5 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:38:35 +0000 Subject: [PATCH 206/225] d --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b69214a8..60059da9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -463,8 +463,7 @@ cmake.args = [ "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /MACHINE:X64", "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", - "-DPython3_LIBRARY=python314t.lib", - "-DCMAKE_GENERATOR_PLATFORM=x64" + "-DPython3_LIBRARY=python314t.lib" ] # "--debug-output", # "--debug-find", From 927d4fc7e7a4c71755879e8fc4bec98fdf0becd1 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:39:02 +0000 Subject: [PATCH 207/225] 3 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 60059da9..4ed33191 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -457,6 +457,7 @@ cmake.args = [ "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPYTHON_NO_LINK", "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPYTHON_NO_LINK", + "-DCMAKE_SYSTEM_PROCESSOR=AMD64", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /MACHINE:X64", From b658f4690ff9d4c4dd918243fd7abb9c042b3740 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:45:20 +0000 Subject: [PATCH 208/225] faster --- .github/workflows/packaging_wheels.yml | 10 ++++++++-- pyproject.toml | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index d1723943..5f399df5 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -109,8 +109,14 @@ jobs: shell: cmd run: | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( - set VSCMD_ARG_TGT_ARCH=x64 - call "%%i\Common7\Tools\VsDevCmd.bat" + echo Setting up x64 build environment... + call "%%i\Common7\Tools\VsDevCmd.bat" -arch=x64 -host_arch=x64 + echo Verifying compiler paths: + where cl + cl 2>&1 | findstr "Version" + echo VSCMD_ARG_TGT_ARCH=%VSCMD_ARG_TGT_ARCH% + echo VCINSTALLDIR=%VCINSTALLDIR% + echo Platform=%Platform% uv run --no-sync python -m cibuildwheel --output-dir wheelhouse ) env: diff --git a/pyproject.toml b/pyproject.toml index 4ed33191..5edea624 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -464,7 +464,10 @@ cmake.args = [ "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /MACHINE:X64", "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", - "-DPython3_LIBRARY=python314t.lib" + "-DPython3_LIBRARY=python314t.lib", + "-DCORE_EXTENSIONS=", + "-DDISABLE_UNITY=ON", + "-DBUILD_UNITTESTS=OFF" ] # "--debug-output", # "--debug-find", From b2d689d2528f63637fcdd1895f989d4b59956f5a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:49:12 +0000 Subject: [PATCH 209/225] v --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5edea624..2095e571 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -460,8 +460,8 @@ cmake.args = [ "-DCMAKE_SYSTEM_PROCESSOR=AMD64", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", - "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /MACHINE:X64", - "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /MACHINE:X64", + "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", "-DPython3_LIBRARY=python314t.lib", From 23b44bdb7f4b2105eb3731fa605c07eb58ccd1ed Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 13:58:12 +0000 Subject: [PATCH 210/225] a --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2095e571..40060b31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -460,8 +460,9 @@ cmake.args = [ "-DCMAKE_SYSTEM_PROCESSOR=AMD64", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", - "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", - "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /LIBPATH:${Python3_LIBRARY_DIRS}", + "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /LIBPATH:${Python3_LIBRARY_DIRS}", + "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}", "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", "-DPython3_LIBRARY=python314t.lib", From d4efd135799ff9709f6d56da3c632ea4e4b0fa8a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 14:05:12 +0000 Subject: [PATCH 211/225] hardcode --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 40060b31..dc57a460 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -460,8 +460,8 @@ cmake.args = [ "-DCMAKE_SYSTEM_PROCESSOR=AMD64", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", - "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /LIBPATH:${Python3_LIBRARY_DIRS}", - "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib /DEFAULTLIB:python314t.lib /LIBPATH:${Python3_LIBRARY_DIRS}", + "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}", "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", From 28296464ea9fee2abafaa8fbc862a2070e49c937 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 14:06:11 +0000 Subject: [PATCH 212/225] vcvars --- .github/workflows/packaging_wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 5f399df5..85faf3df 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -110,11 +110,10 @@ jobs: run: | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( echo Setting up x64 build environment... - call "%%i\Common7\Tools\VsDevCmd.bat" -arch=x64 -host_arch=x64 + call "%%i\VC\Auxiliary\Build\vcvars64.bat" echo Verifying compiler paths: where cl cl 2>&1 | findstr "Version" - echo VSCMD_ARG_TGT_ARCH=%VSCMD_ARG_TGT_ARCH% echo VCINSTALLDIR=%VCINSTALLDIR% echo Platform=%Platform% uv run --no-sync python -m cibuildwheel --output-dir wheelhouse From 1cbe999d8f86f2306fb7beb0a411a93928b0d57b Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 14:22:39 +0000 Subject: [PATCH 213/225] unity --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dc57a460..9e068396 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -467,7 +467,6 @@ cmake.args = [ "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", "-DPython3_LIBRARY=python314t.lib", "-DCORE_EXTENSIONS=", - "-DDISABLE_UNITY=ON", "-DBUILD_UNITTESTS=OFF" ] # "--debug-output", From 630049680fb507c9e2b0907be5f78790e39b66ef Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 14:32:23 +0000 Subject: [PATCH 214/225] linux spot --- .github/workflows/packaging_wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 85faf3df..c2777616 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,8 +34,8 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: - - { os: windows-2025, arch: amd64, cibw_system: win } - #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + #- { os: windows-2025, arch: amd64, cibw_system: win } + - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } @@ -114,7 +114,7 @@ jobs: echo Verifying compiler paths: where cl cl 2>&1 | findstr "Version" - echo VCINSTALLDIR=%VCINSTALLDIR% + echo VCINSTALLDIR=%VCINSTALLDIR%T echo Platform=%Platform% uv run --no-sync python -m cibuildwheel --output-dir wheelhouse ) From e37d05710b97cb20142456c01b4b910c4fe97577 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 14:47:04 +0000 Subject: [PATCH 215/225] debug --- .github/workflows/packaging_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index c2777616..a3b8a45f 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -113,7 +113,6 @@ jobs: call "%%i\VC\Auxiliary\Build\vcvars64.bat" echo Verifying compiler paths: where cl - cl 2>&1 | findstr "Version" echo VCINSTALLDIR=%VCINSTALLDIR%T echo Platform=%Platform% uv run --no-sync python -m cibuildwheel --output-dir wheelhouse From 1bd13ae743f3bf7c5650f4bec5eb9e6e5092375f Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 14:50:38 +0000 Subject: [PATCH 216/225] windows --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a3b8a45f..b1e78997 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -34,8 +34,8 @@ jobs: matrix: python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: - #- { os: windows-2025, arch: amd64, cibw_system: win } - - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + - { os: windows-2025, arch: amd64, cibw_system: win } + #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } #- { os: macos-15, arch: arm64, cibw_system: macosx } #- { os: macos-15, arch: universal2, cibw_system: macosx } From 6b97a4c238c2eb9f6e884416fbabe11cc6a524f1 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 14:56:42 +0000 Subject: [PATCH 217/225] right param --- pyproject.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9e068396..b05c2179 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,8 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", + "pytest-randomly", # Ensures non-deterministic test order + "pytest-run-parallel", # For free-threading testing "mypy", "coverage", "gcovr; python_version < '3.14'", @@ -455,8 +457,8 @@ cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", - "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPYTHON_NO_LINK", - "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPYTHON_NO_LINK", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB", "-DCMAKE_SYSTEM_PROCESSOR=AMD64", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", @@ -465,7 +467,7 @@ cmake.args = [ "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}", "-DPython3_FIND_STRATEGY=LOCATION", "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", - "-DPython3_LIBRARY=python314t.lib", + "-DPython3_LIBRARY_RELEASE=python314t.lib", "-DCORE_EXTENSIONS=", "-DBUILD_UNITTESTS=OFF" ] From 1e0ed9df4c8839cec088afe0dd5118e13c905bbb Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 15:12:37 +0000 Subject: [PATCH 218/225] Flag --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b05c2179..f493455d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -457,8 +457,8 @@ cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", - "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB", - "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED", "-DCMAKE_SYSTEM_PROCESSOR=AMD64", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", From cc947665baf333b1c2882776602bd0a62ab6ec35 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 15:30:35 +0000 Subject: [PATCH 219/225] strip down to repro --- pyproject.toml | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f493455d..80282aa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -446,6 +446,24 @@ before-test = [ ] [[tool.scikit-build.overrides]] +# Regular Windows builds (non-free-threading) +if.platform-system = "^win32" +if.python-version = "!~3.14t" +cmake.args = [ + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_SYSTEM_PROCESSOR=AMD64", + "-DCMAKE_C_COMPILER_LAUNCHER=", + "-DCMAKE_CXX_COMPILER_LAUNCHER=", + "-DCORE_EXTENSIONS=", + "-DBUILD_UNITTESTS=OFF" +] + +[[tool.scikit-build.overrides]] +# Free-threading Windows builds (Python 3.14t) # To get sccache to work, needed a few things # Ninja # /Z7 so we don't write parallel PDBs @@ -453,24 +471,28 @@ before-test = [ # The LIBRARY is because it picks up python314.lib, instead of python314t.lib # TODO: Dynamically add the "t" for future versions (ie: python315t) if.platform-system = "^win32" +if.python-version = "~3.14t" cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED", "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED", - "-DCMAKE_SYSTEM_PROCESSOR=AMD64", "-DCMAKE_C_COMPILER_LAUNCHER=", "-DCMAKE_CXX_COMPILER_LAUNCHER=", - "-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", - "-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", - "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}", - "-DPython3_FIND_STRATEGY=LOCATION", - "-DPython3_FIND_ABI=ANY;ANY;ANY;ON", - "-DPython3_LIBRARY_RELEASE=python314t.lib", - "-DCORE_EXTENSIONS=", - "-DBUILD_UNITTESTS=OFF" ] + # "-DCMAKE_SYSTEM_PROCESSOR=AMD64", + + #"-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + #"-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + + # "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}", + + #"-DPython3_FIND_STRATEGY=LOCATION", + #"-DPython3_FIND_ABI=ANY;ANY;ANY;ON", + #"-DPython3_LIBRARY_RELEASE=python314t.lib", + # "-DCORE_EXTENSIONS=", + # "-DBUILD_UNITTESTS=OFF" # "--debug-output", # "--debug-find", # "--debug-find-pkg=Python3", From 223b749f57b33553987a527282ad7a8564fb6093 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 15:35:57 +0000 Subject: [PATCH 220/225] d --- .github/workflows/packaging_wheels.yml | 4 ++-- pyproject.toml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b1e78997..bcaa2cd5 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -55,7 +55,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run -v pytest -n auto ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" @@ -113,7 +113,7 @@ jobs: call "%%i\VC\Auxiliary\Build\vcvars64.bat" echo Verifying compiler paths: where cl - echo VCINSTALLDIR=%VCINSTALLDIR%T + echo VCINSTALLDIR=%VCINSTALLDIR% echo Platform=%Platform% uv run --no-sync python -m cibuildwheel --output-dir wheelhouse ) diff --git a/pyproject.toml b/pyproject.toml index 80282aa9..c3005270 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -235,6 +235,7 @@ test = [ # dependencies used for running tests "pytest-reraise", "pytest-timeout", "pytest-randomly", # Ensures non-deterministic test order + "pytest-xdist", # Parallel tests (separate processes) "pytest-run-parallel", # For free-threading testing "mypy", "coverage", @@ -471,7 +472,8 @@ cmake.args = [ # The LIBRARY is because it picks up python314.lib, instead of python314t.lib # TODO: Dynamically add the "t" for future versions (ie: python315t) if.platform-system = "^win32" -if.python-version = "~3.14t" +if.python-version = "==3.14.*" +if.implementation = "cpython-freethreading" cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", From 3a99e4157dbd73d275914698ab48bbb1941701c6 Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 15:39:17 +0000 Subject: [PATCH 221/225] impl --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c3005270..dcbbe365 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -449,7 +449,7 @@ before-test = [ [[tool.scikit-build.overrides]] # Regular Windows builds (non-free-threading) if.platform-system = "^win32" -if.python-version = "!~3.14t" +if.implementation = "!cpython-freethreading" cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", From 46565d4562c29e355b34c69444ca33769f1f0b4d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 15:48:30 +0000 Subject: [PATCH 222/225] abi-flags --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dcbbe365..ad3826b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -449,7 +449,7 @@ before-test = [ [[tool.scikit-build.overrides]] # Regular Windows builds (non-free-threading) if.platform-system = "^win32" -if.implementation = "!cpython-freethreading" +if.abi-flags = "" cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", @@ -472,8 +472,7 @@ cmake.args = [ # The LIBRARY is because it picks up python314.lib, instead of python314t.lib # TODO: Dynamically add the "t" for future versions (ie: python315t) if.platform-system = "^win32" -if.python-version = "==3.14.*" -if.implementation = "cpython-freethreading" +if.abi-flags = "t" cmake.args = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", From 2ed610f6872f3eaff480d117d187987161583c5d Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 16:06:19 +0000 Subject: [PATCH 223/225] bigger matrix --- .github/workflows/packaging_wheels.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index bcaa2cd5..8dd78a16 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -32,14 +32,14 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t + python: [ cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: - { os: windows-2025, arch: amd64, cibw_system: win } - #- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - #- { os: macos-15, arch: arm64, cibw_system: macosx } - #- { os: macos-15, arch: universal2, cibw_system: macosx } - #- { os: macos-13, arch: x86_64, cibw_system: macosx } + - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + - { os: macos-15, arch: arm64, cibw_system: macosx } + - { os: macos-15, arch: universal2, cibw_system: macosx } + - { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: @@ -55,6 +55,7 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > + sccache --show-stats && uv run -v pytest -n auto ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} From 4d04875197802ea795daf383b20f4caa702bdc2a Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 16:23:53 +0000 Subject: [PATCH 224/225] Run test in serial --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 8dd78a16..f99d6096 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -56,7 +56,7 @@ jobs: uv pip install -r pylock.toml CIBW_TEST_COMMAND: > sccache --show-stats && - uv run -v pytest -n auto ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs SCCACHE_GHA_ENABLED: "on" SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} ACTIONS_CACHE_SERVICE_V2: "1" From bd4e5eeb2160573f2235438194576f5a20c5b8aa Mon Sep 17 00:00:00 2001 From: Paul Timmins Date: Fri, 12 Sep 2025 16:36:13 +0000 Subject: [PATCH 225/225] Dont make things worse --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad3826b9..31413cab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,9 +234,9 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", - "pytest-randomly", # Ensures non-deterministic test order - "pytest-xdist", # Parallel tests (separate processes) - "pytest-run-parallel", # For free-threading testing + # "pytest-randomly", # Ensures non-deterministic test order + # "pytest-xdist", # Parallel tests (separate processes) + # "pytest-run-parallel", # For free-threading testing "mypy", "coverage", "gcovr; python_version < '3.14'",