Hotfix: use system-provided cyrus-sasl/libsasl2 at runtime (#10)

- Hotfix: use system-provided cyrus-sasl/libsasl2 at runtime
  - revert previous approach for universal build that is now obsoleted by G-Research/librdkafka#9
  - use `librdkafka.redist` version `2.11.0.1-RC1`
- CI: build linux arm64 wheels too
- CI: check version consistency
- Bump version to `2.11.0.1-RC1+gr`

Author: jgiannuzzi

.github/workflows/package.yml

@@ -6,16 +6,23 @@ on:
   push:
 
 env:
-  LIBRDKAFKA_VERSION: v2.11.0-gr
+  LIBRDKAFKA_VERSION: v2.11.0.1-RC1
 
 jobs:
 
-  build-linux-x64:
-    name: Build wheels for Linux x64
-    runs-on: ubuntu-latest
+  build-linux:
+    name: Build wheels for Linux ${{ matrix.arch }}
+    strategy:
+      matrix:
+        include:
+          - arch: x64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
     env:
       OS_NAME: linux
-      ARCH: x64
+      ARCH: ${{ matrix.arch }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
@@ -55,9 +62,19 @@ jobs:
   create_release_artifacts:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
     name: Create release artifacts
-    needs: [build-linux-x64, build-windows]
+    needs: [build-linux, build-windows]
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - name: Check version consistency
+        run: |
+          tag_version=${GITHUB_REF/refs\/tags\/v/}
+          proj_version=$(uv run --no-project --with poetry poetry version -s)
+          if [ "$tag_version" != "$proj_version" ]; then
+            echo "::error title=Version mismatch::Tag version \"$tag_version\" does not match project version \"$proj_version\"."
+            exit 1
+          fi
       - uses: actions/download-artifact@v4
         with:
           path: artifacts

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "confluent-kafka"
-version = "2.11.0+gr.2"
+version = "2.11.0.1rc1+gr"
 description = "Confluent's Python client for Apache Kafka"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

setup.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-import ctypes.util
 import os
 from setuptools import setup
 from setuptools import Extension
@@ -17,44 +16,13 @@
 else:
     librdkafka_libname = 'rdkafka'
 
-# Define the default module to build, without external dependencies.
-module_defs = [
-    {
-        'name': 'confluent_kafka.cimpl.nodeps.cimpl',
-        'libname': librdkafka_libname,
-    }
-]
+module = Extension('confluent_kafka.cimpl',
+                   libraries=[librdkafka_libname],
+                   sources=[os.path.join(ext_dir, 'confluent_kafka.c'),
+                            os.path.join(ext_dir, 'Producer.c'),
+                            os.path.join(ext_dir, 'Consumer.c'),
+                            os.path.join(ext_dir, 'Metadata.c'),
+                            os.path.join(ext_dir, 'AdminTypes.c'),
+                            os.path.join(ext_dir, 'Admin.c')])
 
-# Check for GSSAPI support and add the appropriate module definitions.
-if ctypes.util.find_library('rdkafka_sasl2_2'):
-    module_defs.append(
-        {
-            'name': 'confluent_kafka.cimpl.sasl2_2.cimpl',
-            'libname': 'rdkafka_sasl2_2',
-        }
-    )
-if ctypes.util.find_library('rdkafka_sasl2_3'):
-    module_defs.append(
-        {
-            'name': 'confluent_kafka.cimpl.sasl2_3.cimpl',
-            'libname': 'rdkafka_sasl2_3',
-        }
-    )
-
-setup(
-    ext_modules=[
-        Extension(
-            mod_def['name'],
-            libraries=[mod_def['libname']],
-            sources=[
-                os.path.join(ext_dir, 'confluent_kafka.c'),
-                os.path.join(ext_dir, 'Producer.c'),
-                os.path.join(ext_dir, 'Consumer.c'),
-                os.path.join(ext_dir, 'Metadata.c'),
-                os.path.join(ext_dir, 'AdminTypes.c'),
-                os.path.join(ext_dir, 'Admin.c'),
-            ],
-        )
-        for mod_def in module_defs
-    ]
-)
+setup(ext_modules=[module])

src/confluent_kafka/cimpl/__init__.py

@@ -35,8 +35,6 @@ case $OSTYPE in
         # Need to set up env vars (in docker) so that cibuildwheel finds librdkafka.
         lib_dir=dest/runtimes/linux-$ARCH/native
         export CIBW_ENVIRONMENT="CFLAGS=-I\$PWD/dest/build/native/include LDFLAGS=-L\$PWD/$lib_dir LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$PWD/$lib_dir"
-        # Do not include libsasl2 and its dependencies in the resulting wheel.
-        export CIBW_REPAIR_WHEEL_COMMAND="auditwheel repair --exclude libsasl2.so.2 --exclude libsasl2.so.3 -w {dest_dir} {wheel}"
         ;;
     darwin*)
         os=macos

src/confluent_kafka/cimpl/nodeps/__init__.py

@@ -39,26 +39,9 @@ ARCH=${ARCH:-x64}
 
 if [[ $OSTYPE == linux* ]]; then
     # Linux
-    LIBDIR=runtimes/linux-$ARCH/native
-
-    # Copy the librdkafka build with least dependencies to librdkafka.so.1
-    if [[ $ARCH == arm64* ]]; then
-        cp -v $LIBDIR/{librdkafka.so,librdkafka.so.1}
-    else
-        cp -v $LIBDIR/{centos8-librdkafka.so,librdkafka.so.1}
-        # Copy the librdkafka build with sasl2 support to 2 versions:
-        # librdkafka_sasl2_2.so.1 for debian-based distros
-        # librdkafka_sasl2_3.so.1 for rpm-based distros
-        patchelf --set-soname librdkafka_sasl2_2.so.1 --output $LIBDIR/{librdkafka_sasl2_2.so.1,librdkafka.so}
-        patchelf --replace-needed libsasl2.so.3 libsasl2.so.2 $LIBDIR/librdkafka_sasl2_2.so.1
-        ln -s librdkafka_sasl2_2.so.1 $LIBDIR/librdkafka_sasl2_2.so
-        patchelf --set-soname librdkafka_sasl2_3.so.1 --output $LIBDIR/{librdkafka_sasl2_3.so.1,librdkafka.so}
-        ln -s librdkafka_sasl2_3.so.1 $LIBDIR/librdkafka_sasl2_3.so
-    fi
-    for lib in $LIBDIR/librdkafka*.so.1; do
-        echo $lib
-        ldd $lib
-    done
+
+    cp -v runtimes/linux-$ARCH/native/{librdkafka.so,librdkafka.so.1}
+    ldd runtimes/linux-$ARCH/native/librdkafka.so.1
 
 elif [[ $OSTYPE == darwin* ]]; then
     # MacOS X