Update to librdkafka 2.8.0

Updates `librdkafka` to the latest release 2.8.0. A recent commit* added
a function call that is not available in BoringSSL. This requires us
moving away from BoringSSL again. `libssl-dev` now needs to be available
during build time again.

* https://github.com/confluentinc/librdkafka/blame/93877617709eb071a0f4ec7038c54e2764abefc9/src/rdkafka_ssl.c#L1533

Closes swift-server#186

Author: mimischi

.github/workflows/pull_request.yml

@@ -9,7 +9,7 @@ jobs:
     name: Soundness
     uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main
     with:
-      linux_pre_build_command: "apt-get update -y -q && apt-get install -y -q libsasl2-dev"
+      linux_pre_build_command: "apt-get update -y -q && apt-get install -y -q libsasl2-dev libssl-dev"
       license_header_check_project_name: "swift-kafka-client"
 
   unit-tests:
@@ -36,7 +36,7 @@ jobs:
         run: echo "cxx-interop-matrix=$(curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT"
         env:
           MATRIX_LINUX_COMMAND: "curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
-          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q curl jq libsasl2-dev"
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q curl jq libsasl2-dev libssl-dev"
 
   cxx-interop:
     name: Cxx interop

.github/workflows/unit_tests.yml

@@ -90,7 +90,7 @@ jobs:
           COMMAND_OVERRIDE_NIGHTLY_6_1: "swift test ${{ inputs.linux_nightly_6_1_arguments_override }}"
           COMMAND_OVERRIDE_NIGHTLY_MAIN: "swift test ${{ inputs.linux_nightly_main_arguments_override }}"
         run: |
-          apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev
+          apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev libssl-dev
           curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash
     container:
       image: ${{ matrix.swift.image }}

Package.swift

@@ -18,11 +18,13 @@ import PackageDescription
 let rdkafkaExclude = [
     "./librdkafka/src/CMakeLists.txt",
     "./librdkafka/src/Makefile",
+    "./librdkafka/src/README.lz4.md",
     "./librdkafka/src/generate_proto.sh",
     "./librdkafka/src/librdkafka_cgrp_synch.png",
-    "./librdkafka/src/statistics_schema.json",
+    "./librdkafka/src/opentelemetry/metrics.options",
     "./librdkafka/src/rdkafka_sasl_win32.c",
     "./librdkafka/src/rdwin32.h",
+    "./librdkafka/src/statistics_schema.json",
     "./librdkafka/src/win32_config.h",
     // Remove dependency on cURL. Disabling `ENABLE_CURL` and `WITH_CURL` does
     // not appear to prevent processing of the below files, so we have to exclude
@@ -52,7 +54,6 @@ let package = Package(
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-nio.git", from: "2.55.0"),
-        .package(url: "https://github.com/apple/swift-nio-ssl", from: "2.25.0"),
         .package(url: "https://github.com/swift-server/swift-service-lifecycle.git", from: "2.1.0"),
         .package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
         .package(url: "https://github.com/apple/swift-metrics", from: "2.4.1"),
@@ -64,7 +65,7 @@ let package = Package(
         .target(
             name: "Crdkafka",
             dependencies: [
-                .product(name: "NIOSSL", package: "swift-nio-ssl"),
+                "COpenSSL",
                 .product(name: "libzstd", package: "zstd"),
             ],
             exclude: rdkafkaExclude,
@@ -73,7 +74,6 @@ let package = Package(
             cSettings: [
                 // dummy folder, because config.h is included as "../config.h" in librdkafka
                 .headerSearchPath("./custom/config/dummy"),
-                .headerSearchPath("./custom/include"),
                 .headerSearchPath("./librdkafka/src"),
                 .define("_GNU_SOURCE", to: "1"),  // Fix build error for Swift 5.9 onwards
             ],
@@ -98,6 +98,14 @@ let package = Package(
                 "Kafka"
             ]
         ),
+        .systemLibrary(
+             name: "COpenSSL",
+             pkgConfig: "openssl",
+             providers: [
+                 .brew(["libressl"]),
+                 .apt(["libssl-dev"]),
+             ]
+         ),
         .testTarget(
             name: "KafkaTests",
             dependencies: [
@@ -113,14 +121,21 @@ let package = Package(
 )
 
 for target in package.targets {
-    var settings = target.swiftSettings ?? []
-    settings.append(.enableExperimentalFeature("StrictConcurrency=complete"))
-    target.swiftSettings = settings
+    switch target.type {
+    case .regular, .test, .executable:
+        var settings = target.swiftSettings ?? []
+        settings.append(.enableExperimentalFeature("StrictConcurrency=complete"))
+        target.swiftSettings = settings
+    case .macro, .plugin, .system, .binary:
+        break // These targets do not support settings
+    @unknown default:
+        fatalError("Update to handle new target type \(target.type)")
+    }
 }
 
 // ---    STANDARD CROSS-REPO SETTINGS DO NOT EDIT   --- //
 for target in package.targets {
-    if target.type != .plugin {
+    if target.type != .plugin && target.type != .system {
         var settings = target.swiftSettings ?? []
         // https://github.com/swiftlang/swift-evolution/blob/main/proposals/0444-member-import-visibility.md
         settings.append(.enableUpcomingFeature("MemberImportVisibility"))

README.md

@@ -237,9 +237,13 @@ config.securityProtocol = .saslTLS(
 
 ## librdkafka
 
-The Package depends on [the librdkafka library](https://github.com/confluentinc/librdkafka), which is included as a git submodule.
+The Package depends on [the `librdkafka` library](https://github.com/confluentinc/librdkafka), which is included as a git submodule.
 It has source files that are excluded in `Package.swift`.
 
+### Dependencies
+
+`librdkafka` depends on `openssl`, meaning that `libssl-dev` must be present at build time.
+
 ## Development Setup
 
 We provide a Docker environment for this package. This will automatically start a local Kafka server and run the package tests.