Skip to content

Commit fb21deb

Browse files
evertlammertsevertlammerts
committed
Better streamlined release pipeline
1 parent 9e25a7d commit fb21deb

File tree

5 files changed

+320
-189
lines changed

5 files changed

+320
-189
lines changed

‎.github/workflows/cleanup_pypi.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ jobs:
6262
uv sync --only-group pypi --no-install-project
6363
uv run --no-sync python -u -m duckdb_packaging.pypi_cleanup ${{ inputs.dry-run && '--dry' || '' }} \
6464
${{ inputs.environment == 'pypi-prod-nightly' && '--prod' || '--test' }} \
65+
--verbose \
6566
--username "${{ vars.PYPI_CLEANUP_USERNAME }}" \
6667
--max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} 2>&1 | tee cleanup_output
6768

‎.github/workflows/packaging.yml‎

Lines changed: 15 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -63,127 +63,20 @@ defaults:
6363

6464
jobs:
6565
build_sdist:
66-
name: Build sdist
67-
runs-on: ubuntu-24.04
68-
steps:
69-
70-
- name: Checkout DuckDB Python
71-
uses: actions/checkout@v4
72-
with:
73-
ref: ${{ inputs.git-ref }}
74-
fetch-depth: 0
75-
submodules: true
76-
77-
- name: Checkout DuckDB
78-
shell: bash
79-
run: |
80-
cd external/duckdb
81-
git fetch origin
82-
git checkout ${{ inputs.duckdb-git-ref }}
83-
84-
- name: Set OVERRIDE_GIT_DESCRIBE
85-
if: ${{ inputs.set-version != '' }}
86-
run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
87-
88-
- name: Install Astral UV
89-
uses: astral-sh/setup-uv@v6
90-
with:
91-
version: "0.7.14"
92-
python-version: 3.11
93-
94-
- name: Build sdist
95-
run: uv build --sdist
96-
97-
- name: Install sdist
98-
run: |
99-
cd ${{ runner.temp }}
100-
uv venv
101-
uv pip install ${{ github.workspace }}/dist/duckdb-*.tar.gz
102-
103-
- name: Test sdist
104-
if: ${{ inputs.testsuite != 'none' }}
105-
run: |
106-
# install the test requirements
107-
uv export --only-group test --no-emit-project --output-file ${{ runner.temp }}/pylock.toml --quiet
108-
cd ${{ runner.temp }}
109-
uv pip install -r pylock.toml
110-
# run tests
111-
tests_root="${{ github.workspace }}/tests"
112-
tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}"
113-
uv run --verbose pytest $tests_dir --verbose --ignore=${tests_root}/stubs
114-
115-
- uses: actions/upload-artifact@v4
116-
with:
117-
name: sdist
118-
path: dist/*.tar.gz
119-
compression-level: 0
66+
name: Build an sdist and determine versions
67+
uses: ./.github/workflows/packaging_sdist.yml
68+
with:
69+
testsuite: all
70+
git-ref: ${{ github.ref }}
71+
duckdb-git-ref: ${{ inputs.duckdb-sha }}
72+
set-version: ${{ inputs.stable-version }}
12073

12174
build_wheels:
122-
name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
123-
strategy:
124-
fail-fast: false
125-
matrix:
126-
python: [ cp39, cp310, cp311, cp312, cp313 ]
127-
platform:
128-
- { os: windows-2025, arch: amd64, cibw_system: win }
129-
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
130-
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
131-
- { os: macos-15, arch: arm64, cibw_system: macosx }
132-
- { os: macos-15, arch: universal2, cibw_system: macosx }
133-
- { os: macos-13, arch: x86_64, cibw_system: macosx }
134-
minimal:
135-
- ${{ inputs.minimal }}
136-
exclude:
137-
- { minimal: true, python: cp310 }
138-
- { minimal: true, python: cp311 }
139-
- { minimal: true, python: cp312 }
140-
- { minimal: true, platform: { arch: universal2 } }
141-
runs-on: ${{ matrix.platform.os }}
142-
env:
143-
CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
144-
CIBW_TEST_SOURCES: tests
145-
CIBW_BEFORE_TEST: >
146-
uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
147-
uv pip install -r pylock.toml
148-
CIBW_TEST_COMMAND: >
149-
uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
150-
151-
steps:
152-
- name: Checkout DuckDB Python
153-
uses: actions/checkout@v4
154-
with:
155-
ref: ${{ inputs.git-ref }}
156-
fetch-depth: 0
157-
submodules: true
158-
159-
- name: Checkout DuckDB
160-
shell: bash
161-
run: |
162-
cd external/duckdb
163-
git fetch origin
164-
git checkout ${{ inputs.duckdb-git-ref }}
165-
166-
# Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
167-
- name: Set OVERRIDE_GIT_DESCRIBE
168-
if: ${{ inputs.set-version != '' }}
169-
run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
170-
171-
# Install Astral UV, which will be used as build-frontend for cibuildwheel
172-
- uses: astral-sh/setup-uv@v6
173-
with:
174-
version: "0.7.14"
175-
enable-cache: false
176-
cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
177-
178-
- name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
179-
uses: pypa/[email protected]
180-
env:
181-
CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
182-
CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
183-
184-
- name: Upload wheel
185-
uses: actions/upload-artifact@v4
186-
with:
187-
name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
188-
path: wheelhouse/*.whl
189-
compression-level: 0
75+
name: Build and test releases
76+
uses: ./.github/workflows/packaging_wheels.yml
77+
with:
78+
minimal: false
79+
testsuite: all
80+
git-ref: ${{ github.ref }}
81+
duckdb-git-ref: ${{ inputs.duckdb-sha }}
82+
set-version: ${{ inputs.stable-version }}

‎.github/workflows/packaging_sdist.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Sdist packaging
2+
on:
3+
workflow_call:
4+
inputs:
5+
testsuite:
6+
type: string
7+
description: Testsuite to run (none, fast, all)
8+
required: true
9+
default: all
10+
git-ref:
11+
type: string
12+
description: Git ref of the DuckDB python package
13+
required: false
14+
duckdb-git-ref:
15+
type: string
16+
description: Git ref of DuckDB
17+
required: false
18+
set-version:
19+
description: Force version (vX.Y.Z-((rc|post)N))
20+
required: false
21+
type: string
22+
outputs:
23+
package-version:
24+
description: The version of the DuckDB Python package
25+
value: ${{ jobs.build_sdist.outputs.pkg_version }}
26+
duckdb-version:
27+
description: The version of DuckDB that was packaged
28+
value: ${{ jobs.build_sdist.outputs.duckdb_version }}
29+
30+
jobs:
31+
build_sdist:
32+
name: Build sdist
33+
runs-on: ubuntu-24.04
34+
outputs:
35+
pkg_version: ${{ steps.versioning.outputs.pkg_version }}
36+
duckdb_version: ${{ steps.versioning.outputs.duckdb_version }}
37+
steps:
38+
39+
- name: Checkout DuckDB Python
40+
uses: actions/checkout@v4
41+
with:
42+
ref: ${{ inputs.git-ref }}
43+
fetch-depth: 0
44+
submodules: true
45+
46+
- name: Checkout DuckDB
47+
shell: bash
48+
run: |
49+
cd external/duckdb
50+
git fetch origin
51+
git checkout ${{ inputs.duckdb-git-ref }}
52+
53+
- name: Set OVERRIDE_GIT_DESCRIBE
54+
if: ${{ inputs.set-version != '' }}
55+
run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
56+
57+
- name: Install Astral UV
58+
uses: astral-sh/setup-uv@v6
59+
with:
60+
version: "0.7.14"
61+
python-version: 3.11
62+
63+
- name: Build sdist
64+
run: uv build --sdist
65+
66+
- name: Install sdist
67+
run: |
68+
cd ${{ runner.temp }}
69+
uv venv
70+
uv pip install ${{ github.workspace }}/dist/duckdb-*.tar.gz
71+
72+
- name: Test sdist
73+
if: ${{ inputs.testsuite != 'none' }}
74+
run: |
75+
# install the test requirements
76+
uv export --only-group test --no-emit-project --output-file ${{ runner.temp }}/pylock.toml --quiet
77+
cd ${{ runner.temp }}
78+
uv pip install -r pylock.toml
79+
# run tests
80+
tests_root="${{ github.workspace }}/tests"
81+
tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}"
82+
uv run --verbose pytest $tests_dir --verbose --ignore=${tests_root}/stubs
83+
84+
- id: versioning
85+
run: |
86+
cd ${{ runner.temp }}
87+
echo "pkg_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.__version__)' )" >> $GITHUB_OUTPUT
88+
echo "duckdb_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.duckdb_version)' )" >> $GITHUB_OUTPUT
89+
90+
- uses: actions/upload-artifact@v4
91+
with:
92+
name: sdist
93+
path: dist/*.tar.gz
94+
compression-level: 0

‎.github/workflows/packaging_wheels.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Wheels packaging
2+
on:
3+
workflow_call:
4+
inputs:
5+
minimal:
6+
type: boolean
7+
description: Build a minimal set of wheels to do a sanity check
8+
default: false
9+
testsuite:
10+
type: string
11+
description: Testsuite to run (none, fast, all)
12+
required: true
13+
default: all
14+
git-ref:
15+
type: string
16+
description: Git ref of the DuckDB python package
17+
required: false
18+
duckdb-git-ref:
19+
type: string
20+
description: Git ref of DuckDB
21+
required: false
22+
set-version:
23+
description: Force version (vX.Y.Z-((rc|post)N))
24+
required: false
25+
type: string
26+
27+
jobs:
28+
build_wheels:
29+
name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
python: [ cp39, cp310, cp311, cp312, cp313 ]
34+
platform:
35+
- { os: windows-2025, arch: amd64, cibw_system: win }
36+
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
37+
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
38+
- { os: macos-15, arch: arm64, cibw_system: macosx }
39+
- { os: macos-15, arch: universal2, cibw_system: macosx }
40+
- { os: macos-13, arch: x86_64, cibw_system: macosx }
41+
minimal:
42+
- ${{ inputs.minimal }}
43+
exclude:
44+
- { minimal: true, python: cp310 }
45+
- { minimal: true, python: cp311 }
46+
- { minimal: true, python: cp312 }
47+
- { minimal: true, platform: { arch: universal2 } }
48+
runs-on: ${{ matrix.platform.os }}
49+
env:
50+
CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
51+
CIBW_TEST_SOURCES: tests
52+
CIBW_BEFORE_TEST: >
53+
uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
54+
uv pip install -r pylock.toml
55+
CIBW_TEST_COMMAND: >
56+
uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
57+
58+
steps:
59+
- name: Checkout DuckDB Python
60+
uses: actions/checkout@v4
61+
with:
62+
ref: ${{ inputs.git-ref }}
63+
fetch-depth: 0
64+
submodules: true
65+
66+
- name: Checkout DuckDB
67+
shell: bash
68+
run: |
69+
cd external/duckdb
70+
git fetch origin
71+
git checkout ${{ inputs.duckdb-git-ref }}
72+
73+
# Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
74+
- name: Set OVERRIDE_GIT_DESCRIBE
75+
if: ${{ inputs.set-version != '' }}
76+
run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
77+
78+
# Install Astral UV, which will be used as build-frontend for cibuildwheel
79+
- uses: astral-sh/setup-uv@v6
80+
with:
81+
version: "0.7.14"
82+
enable-cache: false
83+
cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
84+
85+
- name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
86+
uses: pypa/[email protected]
87+
env:
88+
CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
89+
CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
90+
91+
- name: Upload wheel
92+
uses: actions/upload-artifact@v4
93+
with:
94+
name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
95+
path: wheelhouse/*.whl
96+
compression-level: 0

0 commit comments

Comments
 (0)