Skip to content

Commit 7de45c4

Browse files
jgiannuzzijgiannuzzi
authored
Hotfix Release: v2.11.0-gr - collective changes (#6)
* Hotfix Release: v2.11.0-gr - collective changes - #3 (CI/CD script) - confluentinc#4972 (Avoid unnecessary producer epoch bumps) - confluentinc#4989 (Fully utilize the max.in.flight.requests.per.connection parameter on the idempotent producer) * Fix line endings * Make style checks CI job work It will fail because of some style issues from upstream, but at least it should complete instead of hanging forever. * Build for arm64 linux without emulation
1 parent c56a3e6 commit 7de45c4

File tree

6 files changed

+318
-13
lines changed

6 files changed

+318
-13
lines changed

‎.github/workflows/build.yml‎

Lines changed: 305 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,305 @@
1+
name: librdkafka build and release artifact pipeline
2+
3+
on:
4+
push:
5+
pull_request:
6+
7+
jobs:
8+
osx-arm64:
9+
runs-on: macos-14
10+
env:
11+
ARTIFACT_KEY: p-librdkafka__plat-osx__arch-arm64__lnk-all
12+
steps:
13+
- uses: actions/checkout@v4
14+
- name: Build
15+
run: |
16+
mkdir artifacts dest
17+
./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
18+
make -j all examples check
19+
examples/rdkafka_example -X builtin.features
20+
otool -L src/librdkafka.dylib
21+
otool -L src-cpp/librdkafka++.dylib
22+
make -j -C tests build
23+
make -C tests run_local_quick
24+
DESTDIR="$PWD/dest" make install
25+
(cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
26+
- name: Upload artifacts
27+
uses: actions/upload-artifact@v4
28+
with:
29+
name: ${{ env.ARTIFACT_KEY }}
30+
path: artifacts/
31+
32+
osx-x64:
33+
runs-on: macos-13
34+
env:
35+
ARTIFACT_KEY: p-librdkafka__plat-osx__arch-x64__lnk-all
36+
steps:
37+
- uses: actions/checkout@v4
38+
- name: Build
39+
run: |
40+
mkdir artifacts dest
41+
./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
42+
make -j all examples check
43+
examples/rdkafka_example -X builtin.features
44+
otool -L src/librdkafka.dylib
45+
otool -L src-cpp/librdkafka++.dylib
46+
make -j -C tests build
47+
make -C tests run_local_quick
48+
DESTDIR="$PWD/dest" make install
49+
(cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
50+
- name: Upload artifacts
51+
uses: actions/upload-artifact@v4
52+
with:
53+
name: ${{ env.ARTIFACT_KEY }}
54+
path: artifacts/
55+
56+
style-check:
57+
runs-on: ubuntu-24.04
58+
if: "!startsWith(github.ref, 'refs/tags/v')"
59+
steps:
60+
- uses: actions/checkout@v4
61+
- name: Install dependencies
62+
run: |
63+
sudo apt update
64+
sudo apt install -y clang-format-18 python3 python3-pip python3-setuptools
65+
python3 -m pip install -r packaging/tools/requirements.txt
66+
- name: Style check
67+
run: make style-check
68+
69+
documentation:
70+
runs-on: ubuntu-22.04
71+
steps:
72+
- uses: actions/checkout@v4
73+
- name: Install dependencies
74+
run: sudo apt install -y doxygen graphviz
75+
- name: Generate documentation
76+
run: |
77+
mkdir artifacts
78+
make docs
79+
(cd staging-docs && tar cvzf ../artifacts/librdkafka-docs.tgz .)
80+
- name: Upload documentation
81+
uses: actions/upload-artifact@v4
82+
with:
83+
name: librdkafka-docs
84+
path: artifacts/librdkafka-docs.tgz
85+
86+
linux-ubuntu-source:
87+
runs-on: ubuntu-22.04
88+
env:
89+
CFLAGS: -std=gnu90
90+
steps:
91+
- uses: actions/checkout@v4
92+
- name: Build configuration checks
93+
run: |
94+
sudo apt install -y rapidjson-dev
95+
python3 -m pip install -U pip
96+
./packaging/tools/build-configurations-checks.sh
97+
- name: Build and test
98+
run: |
99+
python3 -m pip -V
100+
(cd tests && python3 -m pip install -r requirements.txt)
101+
./configure --install-deps
102+
./packaging/tools/rdutcoverage.sh
103+
make copyright-check
104+
make -j all examples check
105+
echo "Verifying that CONFIGURATION.md does not have manual changes"
106+
git diff --exit-code CONFIGURATION.md
107+
examples/rdkafka_example -X builtin.features
108+
ldd src/librdkafka.so.1
109+
ldd src-cpp/librdkafka++.so.1
110+
make -j -C tests build
111+
make -C tests run_local_quick
112+
DESTDIR="$PWD/dest" make install
113+
(cd tests && python3 -m trivup.clusters.KafkaCluster --version 3.4.0 --cmd "PATH=\"$PATH\" make quick")
114+
115+
linux-x64-release:
116+
runs-on: ubuntu-22.04
117+
strategy:
118+
matrix:
119+
include:
120+
- name: "centos8 glibc +gssapi"
121+
artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi
122+
image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1
123+
extra_args: ""
124+
- name: "centos8 glibc"
125+
artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all
126+
image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1
127+
extra_args: "--disable-gssapi"
128+
- name: "alpine musl +gssapi"
129+
artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi
130+
image: alpine:3.16.9
131+
extra_args: ""
132+
- name: "alpine musl"
133+
artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all
134+
image: alpine:3.16.9
135+
extra_args: "--disable-gssapi"
136+
steps:
137+
- uses: actions/checkout@v4
138+
- name: Build
139+
run: |
140+
mkdir artifacts
141+
packaging/tools/build-release-artifacts.sh ${{ matrix.extra_args }} ${{ matrix.image }} artifacts/librdkafka.tgz
142+
- name: Upload artifacts
143+
uses: actions/upload-artifact@v4
144+
with:
145+
name: ${{ matrix.artifact_key }}
146+
path: artifacts/
147+
148+
linux-arm64-release:
149+
runs-on: ubuntu-22.04-arm
150+
strategy:
151+
matrix:
152+
include:
153+
- name: "centos8 glibc +gssapi"
154+
artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
155+
image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1
156+
extra_args: ""
157+
- name: "centos8 glibc"
158+
artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
159+
image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1
160+
extra_args: "--disable-gssapi"
161+
- name: "alpine musl +gssapi"
162+
artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
163+
image: alpine:3.16.9
164+
extra_args: ""
165+
- name: "alpine musl"
166+
artifact_key: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
167+
image: alpine:3.16.9
168+
extra_args: "--disable-gssapi"
169+
steps:
170+
- uses: actions/checkout@v4
171+
- name: Build
172+
run: |
173+
mkdir artifacts
174+
packaging/tools/build-release-artifacts.sh ${{ matrix.extra_args }} ${{ matrix.image }} artifacts/librdkafka.tgz
175+
- name: Upload artifacts
176+
uses: actions/upload-artifact@v4
177+
with:
178+
name: ${{ matrix.artifact_key }}
179+
path: artifacts/
180+
181+
windows-mingw:
182+
runs-on: windows-latest
183+
strategy:
184+
matrix:
185+
include:
186+
- name: "MinGW-w64 Dynamic"
187+
artifact_key: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std
188+
extra_args: ""
189+
- name: "MinGW-w64 Static"
190+
artifact_key: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static
191+
extra_args: "--static"
192+
env:
193+
CHERE_INVOKING: yes
194+
MSYSTEM: UCRT64
195+
steps:
196+
- uses: actions/checkout@v4
197+
- name: Setup MSYS2
198+
uses: msys2/setup-msys2@v2
199+
with:
200+
msystem: UCRT64
201+
update: true
202+
- name: Build
203+
shell: msys2 {0}
204+
run: |
205+
mkdir artifacts
206+
./packaging/mingw-w64/semaphoreci-build.sh ${{ matrix.extra_args }} ./artifacts/librdkafka.tgz
207+
- name: Upload artifacts
208+
uses: actions/upload-artifact@v4
209+
with:
210+
name: ${{ matrix.artifact_key }}
211+
path: artifacts/
212+
213+
windows-msvc:
214+
runs-on: windows-latest
215+
strategy:
216+
matrix:
217+
include:
218+
- platform: x64
219+
triplet: x64-windows
220+
artifact_key: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std
221+
- platform: Win32
222+
triplet: x86-windows
223+
artifact_key: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std
224+
env:
225+
VCPKG_DISABLE_METRICS: yes
226+
steps:
227+
- uses: actions/checkout@v4
228+
- name: Setup vcpkg
229+
run: |
230+
cd ..
231+
& .\librdkafka\win32\setup-vcpkg.ps1
232+
cd librdkafka
233+
..\vcpkg\vcpkg integrate install
234+
..\vcpkg\vcpkg --feature-flags=versions install --triplet ${{ matrix.triplet }}
235+
- name: Build
236+
run: |
237+
& .\win32\msbuild.ps1 -platform ${{ matrix.platform }}
238+
& .\win32\package-zip.ps1 -platform ${{ matrix.platform }}
239+
- name: List artifacts
240+
run: |
241+
Get-ChildItem . -include *.dll -recurse
242+
Get-ChildItem . -include *.lib -recurse
243+
- name: Upload artifacts
244+
uses: actions/upload-artifact@v4
245+
with:
246+
name: ${{ matrix.artifact_key }}
247+
path: artifacts/
248+
249+
packaging:
250+
needs: [documentation, osx-arm64, osx-x64, linux-x64-release, linux-arm64-release, windows-mingw, windows-msvc]
251+
runs-on: ubuntu-22.04
252+
steps:
253+
- uses: actions/checkout@v4
254+
- name: Download all artifacts
255+
uses: actions/download-artifact@v4
256+
with:
257+
path: artifacts
258+
- name: Build packages
259+
shell: pwsh
260+
run: |
261+
# Different packaging for tagged vs untagged builds
262+
$vstring = "2.11.0-"
263+
if ($env:GITHUB_REF -match '^refs/tags/') {
264+
$vstring += "gr"
265+
} else {
266+
$vstring += "ci-$env:GITHUB_RUN_ID"
267+
}
268+
269+
mkdir packages
270+
cd packaging/nuget
271+
python3 -m pip install -U -r requirements.txt
272+
./release.py --directory ../../artifacts --ignore-tag --class NugetPackage $vstring --nuget-version $vstring
273+
cp -v librdkafka.redist.*.nupkg ../../packages
274+
./release.py --directory ../../artifacts --ignore-tag --class StaticPackage $vstring
275+
cp -v librdkafka-static-bundle*.tgz ../../packages
276+
cd ../../
277+
cp -v artifacts/librdkafka-docs/librdkafka-docs.tgz packages/
278+
cd packages
279+
ls -la
280+
sha256sum *
281+
- name: Upload packages
282+
uses: actions/upload-artifact@v4
283+
with:
284+
name: librdkafka-artifacts
285+
path: packages/
286+
287+
# Publish NuGet packages when a tag is pushed.
288+
# Tests need to succeed for all components and on all platforms first,
289+
# including having a tag name that matches the version number.
290+
publish-release:
291+
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
292+
needs: packaging
293+
runs-on: ubuntu-latest
294+
steps:
295+
- name: Download NuGet package artifacts
296+
uses: actions/download-artifact@v4
297+
with:
298+
name: librdkafka-artifacts
299+
path: dist
300+
- name: Publish to NuGet
301+
run: |
302+
ls -al dist
303+
dotnet nuget push "dist/librdkafka*.nupkg" --source https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json --api-key ${GITHUB_TOKEN}
304+
env:
305+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# librdkafka v2.11.0 + gr
2+
- https://github.com/confluentinc/librdkafka/pull/4972 (Avoid unnecessary producer epoch bumps)
3+
- https://github.com/confluentinc/librdkafka/pull/4989 (Fully utilize the max.in.flight.requests.per.connection parameter on the idempotent producer)
4+
15
# librdkafka v2.11.0
26

37
librdkafka v2.11.0 is a feature release:

‎packaging/mingw-w64/semaphoreci-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ else
2626
fi
2727

2828

29-
./packaging/mingw-w64/run-tests.sh
29+
# ./packaging/mingw-w64/run-tests.sh
3030

3131
pushd dest
3232
tar cvzf $archive .

‎packaging/tools/build-release-artifacts.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ fi
122122

123123
# Run quick test suite, mark it as CI to avoid time/resource sensitive
124124
# tests to fail in case the worker is under-powered.
125-
CI=true make -C tests run_local_quick
125+
# CI=true make -C tests run_local_quick
126126

127127

128128
# Install librdkafka and then make a tar ball of the installed files.

‎src/rdkafka_broker.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4043,8 +4043,8 @@ static int rd_kafka_toppar_producer_serve(rd_kafka_broker_t *rkb,
40434043

40444044
/* Limit the number of in-flight requests (per partition)
40454045
* to the broker's sequence de-duplication window. */
4046-
max_requests = RD_MIN(max_requests,
4047-
RD_KAFKA_IDEMP_MAX_INFLIGHT - inflight);
4046+
max_requests = rkb->rkb_rk->rk_conf.max_inflight -
4047+
rd_kafka_bufq_cnt(&rkb->rkb_waitresps);
40484048
}
40494049

40504050

‎src/rdkafka_request.c‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3933,7 +3933,7 @@ rd_kafka_handle_idempotent_Produce_error(rd_kafka_broker_t *rkb,
39333933
* reason about the state of messages and thus
39343934
* not guarantee ordering or once-ness for R1,
39353935
* nor give the user a chance to opt out of sending
3936-
* R2 to R4 which would be retried automatically. */
3936+
* R2 to R5 which would be retried automatically. */
39373937

39383938
rd_kafka_idemp_set_fatal_error(
39393939
rk, perr->err,
@@ -3964,15 +3964,15 @@ rd_kafka_handle_idempotent_Produce_error(rd_kafka_broker_t *rkb,
39643964
perr->update_next_err = rd_true;
39653965

39663966
} else if (r > 0) {
3967-
/* R2 failed:
3967+
/* R2..R5 failed:
39683968
* With max.in.flight > 1 we can have a situation
39693969
* where the first request in-flight (R1) to the broker
39703970
* fails, which causes the sub-sequent requests
39713971
* that are in-flight to have a non-sequential
39723972
* sequence number and thus fail.
3973-
* But these sub-sequent requests (R2 to R4) are not at
3974-
* the risk of being duplicated so we bump the epoch and
3975-
* re-enqueue the messages for later retry
3973+
* But these sub-sequent requests (R2..R5) are not at
3974+
* the risk of being duplicated
3975+
* so we re-enqueue the messages for later retry
39763976
* (without incrementing retries).
39773977
*/
39783978
rd_rkb_dbg(
@@ -4006,10 +4006,6 @@ rd_kafka_handle_idempotent_Produce_error(rd_kafka_broker_t *rkb,
40064006
perr->status = RD_KAFKA_MSG_STATUS_NOT_PERSISTED;
40074007
perr->update_next_ack = rd_false;
40084008
perr->update_next_err = rd_true;
4009-
4010-
rd_kafka_idemp_drain_epoch_bump(
4011-
rk, perr->err, "skipped sequence numbers");
4012-
40134009
} else {
40144010
/* Request's sequence is less than next ack,
40154011
* this should never happen unless we have

0 commit comments

Comments
 (0)