Skip to content

Commit 3990a2f

Browse files
edenhilledenhill
committed
Travis: restructure wheel building
1 parent 97f1e7a commit 3990a2f

File tree

5 files changed

+221
-31
lines changed

5 files changed

+221
-31
lines changed

‎.travis.yml‎

Lines changed: 63 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
env:
22
global:
3-
- LIBRDKAFKA_VERSION=v1.6.0-RC1
3+
- LIBRDKAFKA_VERSION=v1.6.0-RC2
4+
45
jobs:
56
include:
67
- name: "Source package verification with Python 2.7 (Linux)"
@@ -10,65 +11,96 @@ jobs:
1011
python: "2.7"
1112
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1213
services: docker
14+
1315
- name: "Source package verification with Python 3.6 (Linux)"
1416
os: linux
1517
language: python
1618
dist: xenial
1719
python: "3.6"
1820
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1921
services: docker
22+
2023
- name: "Source package verification with Python 2.7 (OSX)"
2124
os: osx
2225
python: "2.7"
2326
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" INTERPRETER_VERSION="2.7.17"
27+
2428
- name: "Source package verification with Python 3.6 (OSX) +docs"
2529
os: osx
2630
python: "3.6"
2731
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" INTERPRETER_VERSION="3.6.5"
28-
- name: "cibuildwheel (OSX)"
29-
os: osx
30-
env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib" INTERPRETER_VERSION="2.7.17"
31-
- name: "cibuildwheel (manylinux)"
32-
os: linux
33-
dist: xenial
34-
env:
35-
- CIBW_BEFORE_BUILD="tools/prepare-cibuildwheel-linux.sh ${LIBRDKAFKA_VERSION}"
36-
- PYTHON_CONFIGURE_OPTS="--enable-unicode=ucs4 --with-wide-unicode"
37-
- CIBW_MANYLINUX_X86_64_IMAGE="manylinux2010"
38-
- CIBW_MANYLINUX_I686_IMAGE="manylinux2010"
32+
33+
- name: "Wheels: Windows x64"
34+
if: tag is present
35+
os: windows
36+
language: shell
37+
env: BUILD_WHEELS=1
38+
before_install:
39+
- choco install python --version 3.8.0
40+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
41+
# make sure it's on PATH as 'python3'
42+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
43+
install:
44+
- bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
45+
script:
46+
- tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
47+
48+
- name: "Wheels: Windows x86"
49+
if: tag is present
50+
os: windows
51+
language: shell
52+
env: BUILD_WHEELS=1
53+
before_install:
54+
- choco install python --version 3.8.0
55+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
56+
# make sure it's on PATH as 'python3'
57+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
58+
install:
59+
- bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
60+
script:
61+
- tools/wheels/build-wheels.bat x86 win32 dest wheelhouse
62+
63+
- name: "Wheels: Linux x64"
64+
if: tag is present
3965
language: python
40-
python: "2.7"
66+
python: "3.8"
4167
services: docker
68+
env: BUILD_WHEELS=1
69+
script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
70+
71+
- name: "Wheels: MacOSX x64"
72+
if: tag is present
73+
os: osx
74+
language: shell
75+
env: BUILD_WHEELS=1
76+
script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
4277

43-
# See https://cibuildwheel.readthedocs.io/en/latest/options/ for CIBW* vars
4478

4579
# Install test dependencies unconditionally
4680
# Travis OSX envs requires some setup; see tools/prepare-osx.sh
4781
# Install cibuildwheel if this is a tagged PR
4882
before_install:
49-
- if [[ $TRAVIS_OS_NAME == "osx" ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
50-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install -U -r tools/requirements-manylinux.txt --force-reinstall ; fi
83+
- if [[ $TRAVIS_OS_NAME == "osx" && $BUILD_WHEELS != 1 ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
5184

52-
# Install interceptors
53-
# Install librdkafka if not CIBW_BEFORE_BUILD
54-
# Install confluent_kafka[avro] if not CIBW_BEFORE_BUILD
5585
install:
86+
# Install interceptors
5687
- tools/install-interceptors.sh
57-
- pip install -r tests/requirements.txt
58-
- flake8
88+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -r tests/requirements.txt ; fi
5989
- if [[ $MK_DOCS == y ]]; then pip install -r docs/requirements.txt; fi
60-
- if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build && pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] ; fi
90+
# Install librdkafka and confluent_kafka[avro] if not building wheels
91+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build ; fi
92+
6193

62-
# Build wheels
63-
# Make plugins available for tests
64-
# Execute tests if not CIBW_BEFORE_BUILD [osx, linux]
65-
# Execute integration tests if CIBW_BEFORE_BUILD
66-
# Build docs if MK_DOCS
94+
95+
# Note: Will not be run for wheel builds.
6796
script:
68-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then cibuildwheel --output-dir wheelhouse1 && tools/fixup-wheels.sh wheelhouse1 wheelhouse; fi
97+
- flake8
98+
# Build package
99+
- pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf]
69100
- ldd staging/libs/* || otool -L staging/libs/* || true
70-
- if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
71-
- if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-wheels.sh wheelhouse; fi
101+
# Run tests
102+
- if [[ $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
103+
# Build docs
72104
- if [[ $MK_DOCS == y ]]; then make docs; fi
73105

74106
deploy:
@@ -87,4 +119,4 @@ deploy:
87119
on:
88120
repo: confluentinc/confluent-kafka-python
89121
tags: true
90-
condition: "-n $CIBW_BEFORE_BUILD"
122+
condition: "$BUILD_WHEELS == 1"

‎tools/wheels/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Wheel building scripts

‎tools/wheels/build-wheels.bat‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
@echo off
2+
SETLOCAL ENABLEEXTENSIONS
3+
4+
rem x86 or x64
5+
set ARCH=%1
6+
rem win32 or win_amd64
7+
set BW_ARCH=%2
8+
rem librdkafka install destdir (relative path)
9+
set DEST=%3
10+
rem wheelhouse output dir
11+
set WHEELHOUSE=%4
12+
13+
if [%WHEELHOUSE%]==[] goto usage
14+
echo on
15+
16+
set CIBW_BUILD=cp27-%BW_ARCH% cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_WARCH%
17+
set CIBW_BEFORE_BUILD=python -m pip install delvewheel==0.0.6
18+
set CIBW_TEST_REQUIRES=-r tests/requirements.txt
19+
set CIBW_TEST_COMMAND=pytest {project}\tests\test_Producer.py
20+
rem set CIBW_BUILD_VERBOSITY=3
21+
set include=%cd%\%DEST%\build\native\include
22+
set lib=%cd%\%DEST%\build\native\lib\win\%ARCH%\win-%ARCH%-Release\v120
23+
set DLL_DIR=%cd%\%DEST%\runtimes\win-%ARCH%\native
24+
set CIBW_REPAIR_WHEEL_COMMAND=python -m delvewheel repair --add-path %DLL_DIR% -w {dest_dir} {wheel}
25+
26+
set PATH=%PATH%;c:\Program Files\Git\bin\
27+
28+
python -m pip install cibuildwheel==1.7.4 || goto :error
29+
30+
python -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows || goto :error
31+
32+
dir %WHEELHOUSE%
33+
34+
goto :eof
35+
36+
:usage
37+
@echo "Usage: %0 x86|x64 win32|win_amd64 wheelhouse-dir"
38+
exit /B 1
39+
40+
:error
41+
echo Failed with error #%errorlevel%.
42+
exit /b %errorlevel%

‎tools/wheels/build-wheels.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
#
3+
#
4+
# Build wheels (on Linux or OSX) using cibuildwheel.
5+
#
6+
7+
this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
8+
9+
10+
# Skip PyPy, old Python3 versions, and x86 builds.
11+
export CIBW_SKIP="pp* cp35-* *i686"
12+
# Run a simple test suite
13+
export CIBW_TEST_REQUIRES="-r tests/requirements.txt"
14+
export CIBW_TEST_COMMAND="pytest {project}/tests/test_Producer.py"
15+
16+
17+
librdkafka_version=$1
18+
wheeldir=$2
19+
20+
if [[ -z $wheeldir ]]; then
21+
echo "Usage: $0 <librdkafka-nuget-version> <wheeldir>"
22+
exit 1
23+
fi
24+
25+
set -ex
26+
27+
[[ -d $wheeldir ]] || mkdir -p "$wheeldir"
28+
29+
30+
case $OSTYPE in
31+
linux*)
32+
os=linux
33+
# Need to set up env vars (in docker) so that setup.py
34+
# finds librdkafka.
35+
lib_dir=dest/runtimes/linux-x64/native
36+
export CIBW_ENVIRONMENT="CFLAGS=-I\$PWD/dest/build/native/include LDFLAGS=-L\$PWD/$lib_dir LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$PWD/$lib_dir"
37+
;;
38+
darwin*)
39+
os=macos
40+
# Need to set up env vars so that setup.py finds librdkafka.
41+
lib_dir=dest/runtimes/osx-x64/native
42+
export CFLAGS="-I${PWD}/dest/build/native/include"
43+
export LDFLAGS="-L${PWD}/$lib_dir"
44+
;;
45+
*)
46+
echo "$0: Unsupported OSTYPE $OSTYPE"
47+
exit 1
48+
;;
49+
esac
50+
51+
52+
$this_dir/install-librdkafka.sh $librdkafka_version dest
53+
54+
install_pkgs=cibuildwheel==1.7.4
55+
56+
python3 -m pip install $install_pkgs ||
57+
pip3 install $install_pkgs
58+
59+
if [[ -z $TRAVIS ]]; then
60+
cibw_args="--platform $os"
61+
fi
62+
63+
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/$lib_dir python3 -m cibuildwheel --output-dir $wheeldir $cibw_args
64+
65+
ls $wheeldir
66+
67+
for f in $wheeldir/*whl ; do
68+
echo $f
69+
unzip -l $f
70+
done
71+

‎tools/wheels/install-librdkafka.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
VER="$1"
6+
DEST="$2"
7+
8+
if [[ -z $DEST ]]; then
9+
echo "Usage: $0 <librdkafka-redist-version> <destdir>"
10+
exit 1
11+
fi
12+
13+
if [[ -f $DEST/build/native/include/librdkafka/rdkafka.h ]]; then
14+
echo "$0: librdkafka already installed in $DEST"
15+
exit 0
16+
fi
17+
18+
echo "$0: Installing librdkafka $VER to $DEST"
19+
[[ -d "$DEST" ]] || mkdir -p "$DEST"
20+
pushd "$DEST"
21+
22+
curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
23+
24+
unzip lrk$VER.zip
25+
26+
27+
if [[ $OSTYPE == linux* ]]; then
28+
# Linux
29+
30+
# Copy the librdkafka build with least dependencies to librdkafka.so.1
31+
cp -v runtimes/linux-x64/native/{centos6-librdkafka.so,librdkafka.so.1}
32+
ldd runtimes/linux-x64/native/librdkafka.so.1
33+
34+
elif [[ $OSTYPE == darwin* ]]; then
35+
# MacOS X
36+
37+
# Change the library's self-referencing name from
38+
# /Users/travis/.....somelocation/librdkafka.1.dylib to its local path.
39+
install_name_tool -id $PWD/runtimes/osx-x64/native/librdkafka.dylib runtimes/osx-x64/native/librdkafka.dylib
40+
41+
otool -L runtimes/osx-x64/native/librdkafka.dylib
42+
fi
43+
44+
popd

0 commit comments

Comments
 (0)