From 838c55412082c8d63b895915bb61f960ea9a277d Mon Sep 17 00:00:00 2001 From: Jonathan Giannuzzi Date: Wed, 6 Aug 2025 14:15:17 +0100 Subject: [PATCH 1/3] Add sasl2 abi v2 build and include all flavours for linux-arm64 --- .github/workflows/build.yml | 12 +++++-- packaging/nuget/nugetpackage.py | 42 ++++++++++++++++++++-- packaging/tools/build-release-artifacts.sh | 11 ++++++ 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ad5db0644f..9ad3756fb6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,9 +118,13 @@ jobs: matrix: include: - name: "centos8 glibc +gssapi" - artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi + artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi__sasl2abi-v3 image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 extra_args: "" + - name: "centos8 glibc +gssapi +sasl2abi-v2" + artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi__sasl2abi-v2 + image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 + extra_args: "--sasl2-abi-v2" - name: "centos8 glibc" artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all image: quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 @@ -151,9 +155,13 @@ jobs: matrix: include: - name: "centos8 glibc +gssapi" - artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi + artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi__sasl2abi-v3 image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 extra_args: "" + - name: "centos8 glibc +gssapi +sasl2abi-v2" + artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi__sasl2abi-v2 + image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 + extra_args: "--sasl2-abi-v2" - name: "centos8 glibc" artifact_key: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all image: quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 diff --git a/packaging/nuget/nugetpackage.py b/packaging/nuget/nugetpackage.py index ab3655782f..289430b351 100644 --- a/packaging/nuget/nugetpackage.py +++ b/packaging/nuget/nugetpackage.py @@ -71,10 +71,40 @@ class NugetPackage (Package): Mapping({'arch': 'x64', 'plat': 'linux', 'dist': 'centos8', - 'lnk': 'std'}, + 'lnk': 'std', + 'sasl2abi': 'v3'}, 'librdkafka.tgz', './usr/local/lib/librdkafka.so.1', 'runtimes/linux-x64/native/librdkafka.so'), + # Linux glibc centos8 arm64 with GSSAPI + Mapping({'arch': 'arm64', + 'plat': 'linux', + 'dist': 'centos8', + 'lnk': 'std', + 'sasl2abi': 'v3'}, + 'librdkafka.tgz', + './usr/local/lib/librdkafka.so.1', + 'runtimes/linux-arm64/native/librdkafka.so'), + + # Linux glibc centos8 x64 with GSSAPI and SASL2 v2 ABI + Mapping({'arch': 'x64', + 'plat': 'linux', + 'dist': 'centos8', + 'lnk': 'std', + 'sasl2abi': 'v2'}, + 'librdkafka.tgz', + './usr/local/lib/librdkafka.so.1', + 'runtimes/linux-x64/native/sasl2abiv2-librdkafka.so'), + # Linux glibc centos8 arm64 with GSSAPI and SASL2 v2 ABI + Mapping({'arch': 'arm64', + 'plat': 'linux', + 'dist': 'centos8', + 'lnk': 'std', + 'sasl2abi': 'v2'}, + 'librdkafka.tgz', + './usr/local/lib/librdkafka.so.1', + 'runtimes/linux-arm64/native/sasl2abiv2-librdkafka.so'), + # Linux glibc centos8 x64 without GSSAPI (no external deps) Mapping({'arch': 'x64', 'plat': 'linux', @@ -90,7 +120,7 @@ class NugetPackage (Package): 'lnk': 'all'}, 'librdkafka.tgz', './usr/local/lib/librdkafka.so.1', - 'runtimes/linux-arm64/native/librdkafka.so'), + 'runtimes/linux-arm64/native/centos8-librdkafka.so'), # Linux musl alpine x64 without GSSAPI (no external deps) Mapping({'arch': 'x64', @@ -100,6 +130,14 @@ class NugetPackage (Package): 'librdkafka.tgz', './usr/local/lib/librdkafka.so.1', 'runtimes/linux-x64/native/alpine-librdkafka.so'), + # Linux musl alpine arm64 without GSSAPI (no external deps) + Mapping({'arch': 'arm64', + 'plat': 'linux', + 'dist': 'alpine', + 'lnk': 'all'}, + 'librdkafka.tgz', + './usr/local/lib/librdkafka.so.1', + 'runtimes/linux-arm64/native/alpine-librdkafka.so'), # Common Win runtime Mapping({'arch': 'x64', diff --git a/packaging/tools/build-release-artifacts.sh b/packaging/tools/build-release-artifacts.sh index 84e54eb222..6c02fdd17d 100755 --- a/packaging/tools/build-release-artifacts.sh +++ b/packaging/tools/build-release-artifacts.sh @@ -25,6 +25,7 @@ docker_image="" extra_pkgs_rpm="" extra_pkgs_apk="" extra_config_args="" +sasl2_abi="v3" expected_features="gzip snappy ssl sasl regex lz4 sasl_plain sasl_scram plugins zstd sasl_oauthbearer http oidc" # Since cyrus-sasl is the only non-statically-linkable dependency, @@ -42,6 +43,12 @@ else disable_gssapi="" fi +# If --sasl2-abi-v2 is specified, we build librdkafka with the v2 ABI. +if [ "$1" = "--sasl2-abi-v2" ]; then + sasl2_abi="v2" + shift +fi + # Check if we're running on the host or the (docker) build target. if [ "$1" = "--in-docker" -a $# -eq 2 ]; then output="$2" @@ -97,6 +104,10 @@ make -j # Show library linkage (for troubleshooting) and checksums (for verification) for lib in src/librdkafka.so.1 src-cpp/librdkafka++.so.1; do + if [ "$sasl2_abi" = "v2" ]; then + echo "$0: Patching $lib to use libsasl2.so.2" + patchelf --replace-needed libsasl2.so.3 libsasl2.so.2 ${lib} + fi echo "$0: LINKAGE ${lib}:" ldd src/librdkafka.so.1 echo "$0: SHA256 ${lib}:" From fd385e1736abb3d40fe2a1997e914be4ef471e9f Mon Sep 17 00:00:00 2001 From: Jonathan Giannuzzi Date: Wed, 6 Aug 2025 14:52:25 +0100 Subject: [PATCH 2/3] Fix sasl2_abi_v2 argument passing --- packaging/tools/build-release-artifacts.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packaging/tools/build-release-artifacts.sh b/packaging/tools/build-release-artifacts.sh index 6c02fdd17d..dd1d533048 100755 --- a/packaging/tools/build-release-artifacts.sh +++ b/packaging/tools/build-release-artifacts.sh @@ -25,7 +25,6 @@ docker_image="" extra_pkgs_rpm="" extra_pkgs_apk="" extra_config_args="" -sasl2_abi="v3" expected_features="gzip snappy ssl sasl regex lz4 sasl_plain sasl_scram plugins zstd sasl_oauthbearer http oidc" # Since cyrus-sasl is the only non-statically-linkable dependency, @@ -45,8 +44,10 @@ fi # If --sasl2-abi-v2 is specified, we build librdkafka with the v2 ABI. if [ "$1" = "--sasl2-abi-v2" ]; then - sasl2_abi="v2" + sasl2_abi_v2="$1" shift +else + sasl2_abi_v2="" fi # Check if we're running on the host or the (docker) build target. @@ -62,7 +63,7 @@ fi if [ -n "$docker_image" ]; then # Running on the host, spin up the docker builder. - exec docker run -v "$PWD:/v" $docker_image /v/packaging/tools/build-release-artifacts.sh $disable_gssapi --in-docker "/v/$output" + exec docker run -v "$PWD:/v" $docker_image /v/packaging/tools/build-release-artifacts.sh $disable_gssapi $sasl2_abi_v2 --in-docker "/v/$output" # Only reached on exec error exit $? fi @@ -104,7 +105,7 @@ make -j # Show library linkage (for troubleshooting) and checksums (for verification) for lib in src/librdkafka.so.1 src-cpp/librdkafka++.so.1; do - if [ "$sasl2_abi" = "v2" ]; then + if [ -n "$sasl2_abi_v2" ]; then echo "$0: Patching $lib to use libsasl2.so.2" patchelf --replace-needed libsasl2.so.3 libsasl2.so.2 ${lib} fi From c332bca40c84c6025797a95e8ff6060f95e9482e Mon Sep 17 00:00:00 2001 From: Jonathan Giannuzzi Date: Wed, 6 Aug 2025 15:18:40 +0100 Subject: [PATCH 3/3] Compute version from tag or from vcpkg.json --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ad3756fb6..f98075e6d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -267,11 +267,10 @@ jobs: shell: pwsh run: | # Different packaging for tagged vs untagged builds - $vstring = "2.11.0-" - if ($env:GITHUB_REF -match '^refs/tags/') { - $vstring += "gr" + if ($env:GITHUB_REF -match '^refs/tags/v') { + $vstring = $env:GITHUB_REF -replace '^refs/tags/v', '' } else { - $vstring += "ci-$env:GITHUB_RUN_ID" + $vstring = "$((Get-Content .\vcpkg.json | ConvertFrom-Json).version)-ci-$($env:GITHUB_RUN_ID)" } mkdir packages