Skip to content

Commit 4447a0b

Browse files
dkropachevdkropachev
authored
Make building pipeline less confusing (#425)
* cicd/lib-build-and-push: fix python-version description and default * cicd/lib-build-and-push: make choco dont print progress * cicd/lib-build-and-push: use fixed windows version * cicd/lib-build-and-push: allow prerelease python for aarch64 * cicd/lib-build-and-push: use correct `macos-14` instead of `macos-latest` * cicd/lib-build-and-push: load calculated matrix * cicd: move cibuildwheel configuration to pyproject.toml * setup.py: don't panic on wrong command line * cicd/lib-build-and-push: merge aarch64 job to build-wheels * cicd/lib-build-and-push: build pypy and native in single blow
1 parent a226518 commit 4447a0b

File tree

5 files changed

+103
-162
lines changed

5 files changed

+103
-162
lines changed

‎.github/workflows/build-pre-release.yml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,12 @@ on:
1010

1111
target:
1212
type: string
13-
description: "target os to build for: linux,macos,windows"
14-
default: "linux,macos,windows"
15-
16-
cibw-skip:
17-
type: string
18-
description: 'CIBUILDWHEEL builds pattern to skip, goes to CIBW_SKIP env'
19-
required: false
20-
default: 'cp36* cp37* pp*i686 *musllinux*'
13+
description: "target os to build for: linux,macos-x86,macos-arm,windows,linux-aarch64"
14+
default: "linux,macos-x86,macos-arm,windows,linux-aarch64"
2115

2216
jobs:
2317
build-and-publish:
2418
uses: ./.github/workflows/lib-build-and-push.yml
2519
with:
2620
python-version: ${{ inputs.python-version }}
2721
target: ${{ inputs.target }}
28-
cibw-skip: ${{ inputs.cibw-skip }}

‎.github/workflows/build-push.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,3 @@ jobs:
2626
with:
2727
upload: ${{ github.event_name == 'push' && endsWith(github.event.ref, 'scylla') }}
2828
python-version: '3.13'
29-
target: linux,macos,windows

‎.github/workflows/lib-build-and-push.yml‎

Lines changed: 43 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -6,40 +6,20 @@ on:
66
upload:
77
description: 'Upload to PyPI'
88
type: boolean
9+
required: false
910
default: false
1011

1112
python-version:
12-
description: 'Python version to run against'
13-
required: true
13+
description: 'Python version to run on'
1414
type: string
15-
default: "1.13"
15+
required: true
16+
default: "3.13"
1617

1718
target:
18-
type: string
1919
description: "target os to build for: linux,macos,windows"
20-
default: "linux,macos,windows"
21-
22-
cibw-skip:
2320
type: string
24-
description: 'CIBUILDWHEEL builds pattern to skip, goes to CIBW_SKIP env'
2521
required: false
26-
default: 'cp36* cp37* pp*i686 *musllinux*'
27-
28-
env:
29-
CIBW_TEST_COMMAND_LINUX: >
30-
pytest {project}/tests/unit &&
31-
EVENT_LOOP_MANAGER=gevent pytest {project}/tests/unit/io/test_geventreactor.py
32-
33-
CIBW_TEST_COMMAND_MACOS: "pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)' "
34-
CIBW_TEST_COMMAND_WINDOWS: "pytest {project}/tests/unit -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\" "
35-
CIBW_BEFORE_TEST: "pip install -r {project}/test-requirements.txt"
36-
CIBW_BEFORE_BUILD_LINUX: "rm -rf ~/.pyxbld && rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && yum install -y libffi-devel libev libev-devel openssl openssl-devel"
37-
CIBW_ENVIRONMENT: "CASS_DRIVER_BUILD_CONCURRENCY=2 CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST=yes CFLAGS='-g0 -O3'"
38-
CIBW_SKIP: cp36* cp37* pp*i686 *musllinux*
39-
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
40-
CIBW_MANYLINUX_PYPY_X86_64_IMAGE: manylinux_2_28
41-
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
42-
CIBW_MANYLINUX_PYPY_AARCH64_IMAGE: manylinux_2_28
22+
default: "linux,macos-x86,macos-arm,windows,linux-aarch64"
4323

4424
jobs:
4525
prepare-matrix:
@@ -53,19 +33,27 @@ jobs:
5333
run: |
5434
echo -n "[" > /tmp/matrix.json
5535
was_added=""
56-
for os in $(echo "${{ inputs.target }}" | tr -d " " | tr "," "\n")
36+
for target in $(echo "${{ inputs.target }}" | tr -d " " | tr "," "\n")
5737
do
58-
if [[ "${os}" == "linux" ]]; then
38+
if [[ "${target}" == "linux" ]]; then
39+
[ -n "$was_added" ] && echo -n "," >> /tmp/matrix.json
40+
echo -n '{"os":"ubuntu-20.04", "target": "linux"}' >> /tmp/matrix.json
41+
was_added=1
42+
elif [[ "${target}" == "linux-aarch64" ]]; then
5943
[ -n "$was_added" ] && echo -n "," >> /tmp/matrix.json
60-
echo -n '{"os":"ubuntu-20.04","platform":"x86_64"},{"os":"ubuntu-20.04","platform":"PyPy"}' >> /tmp/matrix.json
44+
echo -n '{"os":"ubuntu-20.04", "target": "linux-aarch64"}' >> /tmp/matrix.json
6145
was_added=1
62-
elif [[ "${os}" == "windows" ]]; then
46+
elif [[ "${target}" == "windows" ]]; then
6347
[ -n "$was_added" ] && echo -n "," >> /tmp/matrix.json
64-
echo -n '{"os":"windows-latest","platform":"win64"},{"os":"windows-latest","platform":"PyPy"}' >> /tmp/matrix.json
48+
echo -n '{"os":"windows-2022", "target": "windows"}' >> /tmp/matrix.json
6549
was_added=1
66-
elif [[ "${os}" == "macos" ]]; then
50+
elif [[ "${target}" == "macos-x86" ]]; then
6751
[ -n "$was_added" ] && echo -n "," >> /tmp/matrix.json
68-
echo -n '{"os":"macos-latest","platform":"all"},{"os":"macos-13","platform":"all"},{"os":"macos-latest","platform":"PyPy"}' >> /tmp/matrix.json
52+
echo -n '{"os":"macos-13", "target": "macos-x86"}' >> /tmp/matrix.json
53+
was_added=1
54+
elif [[ "${target}" == "macos-arm" ]]; then
55+
[ -n "$was_added" ] && echo -n "," >> /tmp/matrix.json
56+
echo -n '{"os":"macos-14", "target": "macos-arm"}' >> /tmp/matrix.json
6957
was_added=1
7058
fi
7159
done
@@ -74,32 +62,13 @@ jobs:
7462
echo "matrix=$(cat /tmp/matrix.json)" >> $GITHUB_OUTPUT
7563
7664
build-wheels:
77-
name: Build wheels ${{ matrix.os }} (${{ matrix.platform }})
65+
name: Build wheels for ${{ matrix.target }} on ${{ matrix.os }}
7866
runs-on: ${{ matrix.os }}
67+
needs: prepare-matrix
7968
strategy:
8069
fail-fast: false
8170
matrix:
82-
include:
83-
- os: ubuntu-24.04
84-
platform: x86_64
85-
86-
- os: ubuntu-24.04
87-
platform: PyPy
88-
89-
- os: windows-2022
90-
platform: win64
91-
92-
- os: windows-latest
93-
platform: PyPy
94-
95-
- os: macos-14
96-
platform: all
97-
98-
- os: macos-13
99-
platform: all
100-
101-
- os: macos-latest
102-
platform: PyPy
71+
include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
10372

10473
steps:
10574
- uses: actions/checkout@v4
@@ -122,13 +91,20 @@ jobs:
12291
- name: Install OpenSSL for Windows
12392
if: runner.os == 'Windows'
12493
run: |
125-
choco install openssl --version=3.4.1 -f -y
94+
choco install openssl --version=3.4.1 -f -y --no-progress
95+
96+
- name: Set up QEMU
97+
if: matrix.target == 'linux-aarch64'
98+
uses: docker/setup-qemu-action@v3
99+
with:
100+
platforms: all
101+
image: 'docker.io/tonistiigi/binfmt:desktop-v8.1.5'
126102

127103
- name: Install Conan
128104
if: runner.os == 'Windows'
129105
uses: turtlebrowser/get-conan@main
130106

131-
- name: configure libev for Windows
107+
- name: Configure libev for Windows
132108
if: runner.os == 'Windows'
133109
run: |
134110
conan profile detect
@@ -139,57 +115,31 @@ jobs:
139115
run: |
140116
brew install libev
141117
142-
- name: Overwrite for Linux 64
143-
if: runner.os == 'Linux' && matrix.platform == 'x86_64'
144-
run: |
145-
echo "CIBW_BUILD=cp3*_x86_64" >> $GITHUB_ENV
146-
147-
- name: Overwrite for Linux PyPy
148-
if: runner.os == 'Linux' && matrix.platform == 'PyPy'
118+
- name: Overwrite for MacOS
119+
if: runner.os == 'MacOS'
149120
run: |
150-
echo "CIBW_BUILD=pp*" >> $GITHUB_ENV
151-
echo "CIBW_TEST_COMMAND_LINUX=" >> $GITHUB_ENV
152-
153-
- name: Overwrite for Windows 64
154-
if: runner.os == 'Windows' && matrix.platform == 'win64'
155-
run: |
156-
echo "CIBW_BUILD=cp*win_amd64" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
157-
echo "CIBW_ENVIRONMENT_WINDOWS= CC=clang CXX=clang++" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
158-
159-
- name: Overwrite for Windows PyPY
160-
if: runner.os == 'Windows' && matrix.platform == 'PyPy'
161-
run: |
162-
echo "CIBW_BUILD=pp*" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
163-
echo "CIBW_TEST_COMMAND_WINDOWS=" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
164-
165-
- name: Overwrite for MacOs
166-
if: runner.os == 'MacOs' && matrix.platform == 'all'
167-
run: |
168-
echo "CIBW_BUILD=cp39* cp310* cp311* cp312* cp313*" >> $GITHUB_ENV
169-
echo "CIBW_BEFORE_TEST_MACOS=pip install -r {project}/test-requirements.txt pytest" >> $GITHUB_ENV
121+
##### Set MACOSX_DEPLOYMENT_TARGET
170122
if [ "${{ matrix.os }}" == "macos-13" ]; then
171123
echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV;
172124
echo "Enforcing target deployment for 13.0"
173125
elif [ "${{ matrix.os }}" == "macos-14" ]; then
174126
echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV;
175127
echo "Enforcing target deployment for 14.0"
176-
else
177-
echo "Unknown macos version" && false;
178128
fi
179-
- name: Overwrite for MacOs PyPy
180-
if: runner.os == 'MacOs' && matrix.platform == 'PyPy'
181-
run: |
182-
echo "CIBW_BUILD=pp*" >> $GITHUB_ENV
183-
echo "CIBW_BEFORE_TEST_MACOS=pip install -r {project}/test-requirements.txt pytest" >> $GITHUB_ENV
184-
echo "CIBW_TEST_COMMAND_MACOS=" >> $GITHUB_ENV
185129
186130
- name: Build wheels
131+
if: matrix.target != 'linux-aarch64'
187132
run: |
188133
python3 -m cibuildwheel --output-dir wheelhouse
189134
135+
- name: Build wheels for linux aarch64
136+
if: matrix.target == 'linux-aarch64'
137+
run: |
138+
CIBW_BUILD='cp3*' python -m cibuildwheel --archs aarch64 --output-dir wheelhouse
139+
190140
- uses: actions/upload-artifact@v4
191141
with:
192-
name: wheels-${{ matrix.os }}-${{ matrix.platform }}
142+
name: wheels-${{ matrix.target }}-${{ matrix.os }}
193143
path: ./wheelhouse/*.whl
194144

195145
build-sdist:
@@ -211,56 +161,13 @@ jobs:
211161
name: source-dist
212162
path: dist/*.tar.gz
213163

214-
build-wheels-extra-arch:
215-
# The host should always be linux
216-
runs-on: ubuntu-24.04
217-
name: Build extra arch ${{ matrix.archs }} wheels
218-
strategy:
219-
fail-fast: false
220-
matrix:
221-
archs: [ aarch64,] # ppc64le ]
222-
223-
steps:
224-
- uses: actions/checkout@v4
225-
226-
- name: Set up QEMU
227-
id: qemu
228-
uses: docker/setup-qemu-action@v3
229-
with:
230-
platforms: all
231-
image: 'docker.io/tonistiigi/binfmt:desktop-v8.1.5'
232-
if: runner.os == 'Linux'
233-
234-
- uses: actions/setup-python@v5
235-
name: Install Python
236-
with:
237-
python-version: ${{ inputs.python-version }}
238-
239-
- name: Install cibuildwheel
240-
run: |
241-
python -m pip install cibuildwheel==2.22.0
242-
243-
- name: Build wheels
244-
env:
245-
CIBW_BUILD: "cp39* cp310* cp311* cp312* cp313*" # limit to specific version since it take much more time than jobs limit
246-
run: |
247-
python -m cibuildwheel --archs ${{ matrix.archs }} --output-dir wheelhouse
248-
249-
- uses: actions/upload-artifact@v4
250-
with:
251-
name: wheels-${{ matrix.archs }}
252-
path: ./wheelhouse/*.whl
253-
254164
upload_pypi:
255165
if: inputs.upload
256-
needs: [build-wheels, build-wheels-extra-arch, build-sdist]
166+
needs: [build-wheels, build-sdist]
257167
runs-on: ubuntu-24.04
258168
permissions:
259169
id-token: write
260170

261-
# upload to PyPI on every tag starting with 'v'
262-
# alternatively, to publish when a GitHub Release is created, use the following rule:
263-
# if: github.event_name == 'release' && github.event.action == 'published'
264171
steps:
265172
- uses: actions/download-artifact@v4
266173
with:

‎pyproject.toml‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,4 +76,50 @@ requires = [
7676
"Cython",
7777
]
7878

79-
build-backend = "setuptools.build_meta"
79+
build-backend = "setuptools.build_meta"
80+
81+
[tool.setuptools_scm]
82+
version_file = "cassandra/_version.py"
83+
tag_regex = '(?P<version>\d*?\.\d*?\.\d*?)-scylla'
84+
85+
#### CI BUILDWHEEL CONFIG ####
86+
87+
[tool.cibuildwheel]
88+
build-frontend = "build[uv]"
89+
environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "yes", CFLAGS = "-g0 -O3" }
90+
skip = ["cp2*", "cp36*", "pp36*", "cp37*", "pp37*", "*i686", "*musllinux*"]
91+
build = ["cp3*", "pp3*"]
92+
93+
before-test = "pip install -r {project}/test-requirements.txt"
94+
95+
manylinux-x86_64-image = "manylinux_2_28"
96+
manylinux-aarch64-image = "manylinux_2_28"
97+
manylinux-pypy_x86_64-image = "manylinux_2_28"
98+
manylinux-pypy_aarch64-image = "manylinux_2_28"
99+
100+
[tool.cibuildwheel.linux]
101+
102+
before-build = "rm -rf ~/.pyxbld && rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && yum install -y libffi-devel libev libev-devel openssl openssl-devel"
103+
test-command = [
104+
"pytest {package}/tests/unit",
105+
"EVENT_LOOP_MANAGER=gevent pytest {package}/tests/unit/io/test_geventreactor.py",
106+
]
107+
108+
[tool.cibuildwheel.macos]
109+
build-frontend = "build"
110+
test-command = [
111+
"pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)'"
112+
]
113+
114+
[tool.cibuildwheel.windows]
115+
build-frontend = "build"
116+
test-command = [
117+
"pytest {project}/tests/unit -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\""
118+
]
119+
120+
# TODO: set CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST to yes when https://github.com/scylladb/python-driver/issues/429 is fixed
121+
environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "no"}
122+
123+
[[tool.cibuildwheel.overrides]]
124+
select = "pp*"
125+
test-command = []

‎setup.py‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,22 @@
1414

1515
from __future__ import print_function
1616

17-
import fnmatch
1817
import os
1918
import shutil
2019
import sys
2120
import json
2221
import warnings
2322
from pathlib import Path
24-
from sysconfig import get_config_vars
2523

26-
if __name__ == '__main__' and sys.argv[1] == "gevent_nosetests":
27-
print("Running gevent tests")
28-
from gevent.monkey import patch_all
29-
patch_all()
30-
31-
if __name__ == '__main__' and sys.argv[1] == "eventlet_nosetests":
32-
print("Running eventlet tests")
33-
from eventlet import monkey_patch
34-
monkey_patch()
24+
if __name__ == '__main__' and len(sys.argv) > 1:
25+
if sys.argv[1] == "gevent_nosetests":
26+
print("Running gevent tests")
27+
from gevent.monkey import patch_all
28+
patch_all()
29+
elif sys.argv[1] == "eventlet_nosetests":
30+
print("Running eventlet tests")
31+
from eventlet import monkey_patch
32+
monkey_patch()
3533

3634
from setuptools.command.build_ext import build_ext
3735
from setuptools import Extension, Command, setup
@@ -57,7 +55,7 @@ class eventlet_nosetests(nosetests):
5755
description = "run nosetests with eventlet monkey patching"
5856

5957
has_cqlengine = False
60-
if __name__ == '__main__' and sys.argv[1] == "install":
58+
if __name__ == '__main__' and len(sys.argv) > 1 and sys.argv[1] == "install":
6159
try:
6260
import cqlengine
6361
has_cqlengine = True
@@ -453,9 +451,7 @@ def run_setup(extensions):
453451
else:
454452
sys.stderr.write("Bypassing Cython setup requirement\n")
455453

456-
setup(
457-
tests_require=['nose', 'mock>=2.0.0', 'PyYAML', 'pytz', 'sure'],
458-
**kw)
454+
setup(**kw)
459455

460456

461457
run_setup(None)

0 commit comments

Comments
 (0)