Skip to content

Commit d1b2dc6

Browse files
emasabemasab
authored
Pipeline improvements about machine types, auto-cancel, caching, manual promotions (#5191)
* Pipeline improvements about machine types and auto-cancel * Use cached docker image for integration tests, style checks, docs build * vcpkg cache * msys2 cache * Upgrade macOS agents
1 parent 01ec279 commit d1b2dc6

File tree

13 files changed

+452
-102
lines changed

13 files changed

+452
-102
lines changed

‎.semaphore/run-all-tests.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: run-all-tests
33

44
agent:
55
machine:
6-
type: s1-prod-ubuntu24-04-amd64-1
6+
type: s1-prod-ubuntu24-04-amd64-00
77

88
execution_time_limit:
99
hours: 3
@@ -32,7 +32,7 @@ blocks:
3232
task:
3333
agent:
3434
machine:
35-
type: s1-prod-ubuntu24-04-amd64-2
35+
type: s1-prod-ubuntu24-04-amd64-1
3636
prologue:
3737
commands:
3838
- if [[ "$TEST_ARCHES" != *"x86_64"* ]]; then exit 0; fi
@@ -56,7 +56,7 @@ blocks:
5656
task:
5757
agent:
5858
machine:
59-
type: s1-prod-ubuntu24-04-arm64-2
59+
type: s1-prod-ubuntu24-04-arm64-1
6060
prologue:
6161
commands:
6262
- if [[ "$TEST_ARCHES" != *"aarch64"* ]]; then exit 0; fi

‎.semaphore/semaphore-integration.yml‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
version: v1.0
2+
name: 'librdkafka integration and all architectures tests pipeline'
3+
agent:
4+
machine:
5+
type: s1-prod-ubuntu24-04-arm64-00
6+
execution_time_limit:
7+
hours: 3
8+
9+
auto_cancel:
10+
running:
11+
when: "branch != 'master'"
12+
13+
global_job_config:
14+
prologue:
15+
commands:
16+
- checkout
17+
- mkdir artifacts
18+
- mkdir dest
19+
env_vars:
20+
- name: CACHE_TAG
21+
value: '7'
22+
- name: CI
23+
value: 'true'
24+
- name: KAFKA_VERSION
25+
value: '4.0.0'
26+
- name: CP_VERSION
27+
value: '8.0.0'
28+
# Shows plain output from docker build (no progress bars)
29+
- name: BUILDKIT_PROGRESS
30+
value: plain
31+
32+
blocks:
33+
- name: 'OSX arm64/m1'
34+
dependencies: []
35+
task:
36+
agent:
37+
machine:
38+
type: s1-macos-15-arm64-8
39+
env_vars:
40+
- name: ARTIFACT_KEY
41+
value: p-librdkafka__plat-osx__arch-arm64__lnk-all
42+
epilogue:
43+
commands:
44+
- '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
45+
jobs:
46+
- name: 'Build'
47+
commands:
48+
- ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
49+
- make -j all examples check
50+
- examples/rdkafka_example -X builtin.features
51+
- otool -L src/librdkafka.dylib
52+
- otool -L src-cpp/librdkafka++.dylib
53+
- make -j -C tests build
54+
- make -C tests run_local_quick
55+
- DESTDIR="$PWD/dest" make install
56+
- (cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
57+
58+
59+
- name: 'OSX x64'
60+
dependencies: []
61+
task:
62+
agent:
63+
machine:
64+
type: s1-macos-15-amd64-6
65+
env_vars:
66+
- name: ARTIFACT_KEY
67+
value: p-librdkafka__plat-osx__arch-x64__lnk-all
68+
epilogue:
69+
commands:
70+
- '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
71+
jobs:
72+
- name: 'Build'
73+
commands:
74+
- ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
75+
- make -j all examples check
76+
- examples/rdkafka_example -X builtin.features
77+
- otool -L src/librdkafka.dylib
78+
- otool -L src-cpp/librdkafka++.dylib
79+
- make -j -C tests build
80+
- make -C tests run_local_quick
81+
- DESTDIR="$PWD/dest" make install
82+
- (cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
83+
84+
- name: 'Linux Ubuntu amd64: integration tests'
85+
dependencies: []
86+
task:
87+
agent:
88+
machine:
89+
type: s1-prod-ubuntu24-04-amd64-1
90+
env_vars:
91+
- name: CFLAGS
92+
value: -std=gnu90 # Test minimum C standard, default in CentOS 7
93+
prologue:
94+
commands:
95+
- '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
96+
jobs:
97+
- name: 'Build and integration tests with "classic" protocol'
98+
commands:
99+
- cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG}
100+
- packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG}
101+
./packaging/tools/run-integration-tests.sh
102+
${KAFKA_VERSION} ${CP_VERSION} classic
103+
- cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION}
104+
- name: 'Build and integration tests with "consumer" protocol'
105+
commands:
106+
- cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG}
107+
- packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG}
108+
./packaging/tools/run-integration-tests.sh
109+
${KAFKA_VERSION} ${CP_VERSION} consumer
110+
- cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION}
111+
112+
113+
- name: 'Linux arm64: local quick tests'
114+
dependencies: []
115+
task:
116+
agent:
117+
machine:
118+
type: s1-prod-ubuntu24-04-arm64-1
119+
prologue:
120+
commands:
121+
- '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
122+
epilogue:
123+
commands:
124+
- '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
125+
jobs:
126+
- name: 'Build: centos8 glibc +gssapi'
127+
env_vars:
128+
- name: ARTIFACT_KEY
129+
value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
130+
commands:
131+
- packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
132+
133+
- name: 'Build: centos8 glibc'
134+
env_vars:
135+
- name: ARTIFACT_KEY
136+
value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
137+
commands:
138+
- packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
139+
140+
- name: 'Build: alpine musl +gssapi'
141+
env_vars:
142+
- name: ARTIFACT_KEY
143+
value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
144+
commands:
145+
- packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
146+
147+
- name: 'Build: alpine musl'
148+
env_vars:
149+
- name: ARTIFACT_KEY
150+
value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
151+
commands:
152+
- packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
153+
154+
155+
- name: 'Windows x64: MinGW-w64'
156+
dependencies: []
157+
task:
158+
agent:
159+
machine:
160+
type: s1-prod-windows
161+
env_vars:
162+
- name: CHERE_INVOKING
163+
value: 'yes'
164+
- name: MSYSTEM
165+
value: UCRT64
166+
prologue:
167+
commands:
168+
# Set up msys2
169+
- "& .\\win32\\setup-msys2.ps1"
170+
epilogue:
171+
commands:
172+
- if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ }
173+
jobs:
174+
- name: 'Build: MinGW-w64 Dynamic'
175+
env_vars:
176+
- name: ARTIFACT_KEY
177+
value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std
178+
commands:
179+
- C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh ./artifacts/librdkafka.tgz'
180+
181+
- name: 'Build: MinGW-w64 Static'
182+
env_vars:
183+
- name: ARTIFACT_KEY
184+
value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static
185+
commands:
186+
- C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh --static ./artifacts/librdkafka.tgz'
187+
188+
- name: 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019'
189+
dependencies: []
190+
task:
191+
agent:
192+
machine:
193+
type: s1-prod-windows
194+
env_vars:
195+
# Disable vcpkg telemetry
196+
- name: VCPKG_DISABLE_METRICS
197+
value: 'yes'
198+
prologue:
199+
commands:
200+
# install vcpkg in the parent directory.
201+
- cd ..
202+
# Setup vcpkg
203+
- "& .\\librdkafka\\win32\\setup-vcpkg.ps1"
204+
- cd librdkafka
205+
epilogue:
206+
commands:
207+
- Get-ChildItem . -include *.dll -recurse
208+
- Get-ChildItem . -include *.lib -recurse
209+
- if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ }
210+
jobs:
211+
- name: 'Build: MSVC x64'
212+
env_vars:
213+
- name: triplet
214+
value: x64-windows
215+
- name: ARTIFACT_KEY
216+
value: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std
217+
commands:
218+
- "& .\\win32\\msbuild.ps1 -platform x64"
219+
- "& .\\win32\\package-zip.ps1 -platform x64"
220+
- name: 'Build: MSVC x86'
221+
env_vars:
222+
- name: triplet
223+
value: x86-windows
224+
- name: ARTIFACT_KEY
225+
value: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std
226+
commands:
227+
- "& .\\win32\\msbuild.ps1 -platform Win32"
228+
- "& .\\win32\\package-zip.ps1 -platform Win32"
229+
230+
promotions:
231+
- name: Run all tests on master commits
232+
pipeline_file: run-all-tests.yml
233+
parameters:
234+
env_vars:
235+
- required: true
236+
name: TEST_KAFKA_GIT_REF
237+
default_value: 4.0.0
238+
- required: true
239+
name: TEST_TYPE
240+
default_value: plaintext,ssl
241+
- required: true
242+
name: TEST_ARCHES
243+
default_value: x86_64,aarch64
244+
- required: true
245+
name: TEST_PARALLEL
246+
default_value: "1"
247+
auto_promote_on:
248+
- result: passed
249+
branch:
250+
- "master"

0 commit comments

Comments
 (0)