Skip to content

Commit 6a78a3a

Browse files
PrasanthV454PrasanthV454
authored
Migrate Travis jobs to Semaphore (#1503)
1 parent f3055be commit 6a78a3a

File tree

10 files changed

+240
-123
lines changed

10 files changed

+240
-123
lines changed

‎.semaphore/semaphore.yml‎

Lines changed: 150 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ global_job_config:
1010
prologue:
1111
commands:
1212
- checkout
13+
- mkdir artifacts
1314
blocks:
1415
- name: "Wheels: OSX x64"
1516
run:
@@ -29,7 +30,7 @@ blocks:
2930
commands:
3031
- PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
3132
- tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
32-
- artifact push workflow wheelhouse-macOS-${ARCH}.tgz
33+
- artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
3334
- name: "Wheels: OSX arm64"
3435
run:
3536
when: "tag =~ '.*'"
@@ -50,40 +51,100 @@ blocks:
5051
commands:
5152
- PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
5253
- tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
53-
- artifact push workflow wheelhouse-macOS-${ARCH}.tgz
54-
55-
- name: Source package verification with Python 3 (OSX x64) +docs
54+
- artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
55+
- name: "Wheels: Linux arm64"
56+
run:
57+
when: "tag =~ '.*'"
5658
dependencies: []
5759
task:
5860
agent:
5961
machine:
60-
type: s1-prod-macos
62+
type: s1-prod-ubuntu20-04-arm64-1
6163
env_vars:
6264
- name: OS_NAME
63-
value: osx
65+
value: linux
6466
- name: ARCH
6567
value: arm64
6668
jobs:
6769
- name: Build
6870
commands:
69-
# use a virtualenv
70-
- python3 -m venv _venv && source _venv/bin/activate
71-
- pip install -r docs/requirements.txt
72-
- pip install -U protobuf
73-
# install librdkafka
74-
- lib_dir=dest/runtimes/$OS_NAME-$ARCH/native
75-
- tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest
76-
- export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include"
77-
- export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}"
78-
- export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir"
79-
- export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/$lib_dir"
80-
# install confluent-kafka
81-
- python setup.py build && python setup.py install
82-
- make docs
83-
- name: "Wheels: Linux arm64"
71+
- ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
72+
- tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
73+
- artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
74+
- name: "Wheels: Linux x64"
75+
run:
76+
when: "tag =~ '.*'"
77+
dependencies: []
78+
task:
79+
agent:
80+
machine:
81+
type: s1-prod-ubuntu20-04-amd64-3
82+
env_vars:
83+
- name: OS_NAME
84+
value: linux
85+
- name: ARCH
86+
value: x64
87+
jobs:
88+
- name: Build
89+
commands:
90+
- ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
91+
- tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
92+
- artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
93+
- name: "Wheels: Windows"
8494
run:
8595
when: "tag =~ '.*'"
8696
dependencies: []
97+
task:
98+
agent:
99+
machine:
100+
type: s1-prod-windows
101+
env_vars:
102+
- name: OS_NAME
103+
value: windows
104+
- name: ARCH
105+
value: x64
106+
prologue:
107+
commands:
108+
- cache restore msys2-x64
109+
- ".\\tools\\mingw-w64\\setup-msys2.ps1"
110+
- $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH
111+
- bash -lc './tools/mingw-w64/msys2-dependencies.sh'
112+
- cache delete msys2-x64
113+
- cache store msys2-x64 c:/msys64
114+
jobs:
115+
- name: Build
116+
env_vars:
117+
- name: CHERE_INVOKING
118+
value: 'yes'
119+
- name: MSYSTEM
120+
value: UCRT64
121+
commands:
122+
- bash tools/mingw-w64/semaphore_commands.sh
123+
- bash tools/wheels/install-librdkafka.sh $env:LIBRDKAFKA_VERSION.TrimStart("v") dest
124+
- tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
125+
- tar -czf wheelhouse-windows-${Env:ARCH}.tgz wheelhouse
126+
- artifact push workflow wheelhouse-windows-${Env:ARCH}.tgz --destination artifacts/wheels-${Env:OS_NAME}-${Env:ARCH}.tgz/
127+
- name: "Source package verification and Integration tests with Python 3 (Linux x64)"
128+
dependencies: []
129+
task:
130+
agent:
131+
machine:
132+
type: s1-prod-ubuntu20-04-amd64-2
133+
env_vars:
134+
- name: OS_NAME
135+
value: linux
136+
- name: ARCH
137+
value: x64
138+
jobs:
139+
- name: Build
140+
commands:
141+
- sem-version python 3.8
142+
# use a virtualenv
143+
- python3 -m venv _venv && source _venv/bin/activate
144+
- chmod u+r+x tools/source-package-verification.sh
145+
- tools/source-package-verification.sh
146+
- name: "Source package verification with Python 3 (Linux arm64)"
147+
dependencies: []
87148
task:
88149
agent:
89150
machine:
@@ -96,6 +157,71 @@ blocks:
96157
jobs:
97158
- name: Build
98159
commands:
99-
- ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
100-
- tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
101-
- artifact push workflow wheelhouse-linux-${ARCH}.tgz
160+
- sem-version python 3.8
161+
# use a virtualenv
162+
- python3 -m venv _venv && source _venv/bin/activate
163+
- chmod u+r+x tools/source-package-verification.sh
164+
- tools/source-package-verification.sh
165+
- name: "Source package verification with Python 3 (OSX x64) +docs"
166+
dependencies: []
167+
task:
168+
agent:
169+
machine:
170+
type: s1-prod-macos
171+
env_vars:
172+
- name: OS_NAME
173+
value: osx
174+
- name: ARCH
175+
value: x64
176+
jobs:
177+
- name: Build
178+
commands:
179+
- sem-version python 3.8
180+
# use a virtualenv
181+
- python3 -m venv _venv && source _venv/bin/activate
182+
- chmod u+r+x tools/source-package-verification.sh
183+
- tools/source-package-verification.sh
184+
- name: "Source package verification with Python 3 (OSX arm64) +docs"
185+
dependencies: []
186+
task:
187+
agent:
188+
machine:
189+
type: s1-prod-macos-arm64
190+
env_vars:
191+
- name: OS_NAME
192+
value: osx
193+
- name: ARCH
194+
value: arm64
195+
jobs:
196+
- name: Build
197+
commands:
198+
- sem-version python 3.8
199+
# use a virtualenv
200+
- python3 -m venv _venv && source _venv/bin/activate
201+
- chmod u+r+x tools/source-package-verification.sh
202+
- tools/source-package-verification.sh
203+
- name: "Packaging"
204+
run:
205+
when: "tag =~ '.*'"
206+
dependencies:
207+
- "Wheels: OSX x64"
208+
- "Wheels: OSX arm64"
209+
- "Wheels: Linux arm64"
210+
- "Wheels: Linux x64"
211+
- "Wheels: Windows"
212+
task:
213+
agent:
214+
machine:
215+
type: s1-prod-ubuntu20-04-amd64-3
216+
jobs:
217+
- name: "Packaging all artifacts"
218+
commands:
219+
- artifact pull workflow artifacts
220+
- cd artifacts
221+
- ls *.tgz |xargs -n1 tar -xvf
222+
- tar cvf confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz wheelhouse/
223+
- ls -la
224+
- sha256sum confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz
225+
- cd ..
226+
- artifact push project artifacts/confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz --destination confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz
227+
- echo Thank you

‎.travis.yml‎

Lines changed: 0 additions & 92 deletions
This file was deleted.

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
- Added support for Default num_partitions in CreateTopics Admin API.
99
- Added support for password protected private key in CachedSchemaRegistryClient.
1010
- Add reference support in Schema Registry client. (@RickTalken, #1304)
11+
- Migrated travis jobs to Semaphore CI (#1503)
12+
1113

1214
## v2.0.2
1315

‎tools/mingw-w64/msys2-dependencies.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
export msys2='cmd //C RefreshEnv.cmd '
6+
export msys2+='& set MSYS=winsymlinks:nativestrict '
7+
export msys2+='& C:\\msys64\\msys2_shell.cmd -defterm -no-start'
8+
export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --"
9+
export msys2+=" -msys2 -c "\"\$@"\" --"
10+
11+
# Have to update pacman first or choco upgrade will failure due to migration
12+
# to zstd instead of xz compression
13+
$msys2 pacman -Sy --noconfirm pacman
14+
15+
## Install more MSYS2 packages from https://packages.msys2.org/base here
16+
$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc
17+
18+
## Install unzip
19+
$msys2 pacman --sync --noconfirm --needed unzip

‎tools/mingw-w64/semaphore_commands.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
$msys2 pacman -S python --version 3.8.0
3+
4+
set -e
5+
6+
export PATH="$PATH;C:\Python38;C:\Python38\Scripts"
7+
export MAKE=mingw32-make # so that Autotools can find it
8+
9+
cmd /c mklink /D C:\Python38\python3.exe C:\Python38\python.exe
10+
11+
python -m pip install cibuildwheel==2.12.0

‎tools/mingw-w64/setup-msys2.ps1‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Install (if necessary) and set up msys2.
2+
3+
4+
$url="https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-base-x86_64-20221028.sfx.exe"
5+
$sha256="e365b79b4b30b6f4baf34bd93f3d2a41c0a92801c7a96d79cddbfca1090a0554"
6+
7+
8+
if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) {
9+
echo "Downloading and installing msys2 to c:\msys64"
10+
11+
(New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe')
12+
13+
# Verify checksum
14+
(Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256
15+
16+
# Install msys2
17+
.\msys2-installer.exe -y -oc:\
18+
19+
Remove-Item msys2-installer.exe
20+
21+
# Set up msys2 the first time
22+
echo "Setting up msys"
23+
c:\msys64\usr\bin\bash -lc ' '
24+
25+
} else {
26+
echo "Using previously installed msys2"
27+
}
28+
29+
# Update packages
30+
echo "Updating msys2 packages"
31+
c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'"

0 commit comments

Comments
 (0)