diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 404601cd..afc987e8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -9,7 +9,7 @@ jobs: name: Soundness uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main with: - linux_pre_build_command: "apt-get update -y -q && apt-get install -y -q libsasl2-dev" + linux_pre_build_command: "apt-get update -y -q && apt-get install -y -q libsasl2-dev libssl-dev" license_header_check_project_name: "swift-kafka-client" unit-tests: @@ -36,7 +36,7 @@ jobs: run: echo "cxx-interop-matrix=$(curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT" env: MATRIX_LINUX_COMMAND: "curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash" - MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q curl jq libsasl2-dev" + MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q curl jq libsasl2-dev libssl-dev" cxx-interop: name: Cxx interop diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5f0bdfde..0de454e1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -90,7 +90,7 @@ jobs: COMMAND_OVERRIDE_NIGHTLY_6_1: "swift test ${{ inputs.linux_nightly_6_1_arguments_override }}" COMMAND_OVERRIDE_NIGHTLY_MAIN: "swift test ${{ inputs.linux_nightly_main_arguments_override }}" run: | - apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev + apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev libssl-dev curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash container: image: ${{ matrix.swift.image }} diff --git a/Package.swift b/Package.swift index 0babb3d4..0fe43364 100644 --- a/Package.swift +++ b/Package.swift @@ -18,11 +18,13 @@ import PackageDescription let rdkafkaExclude = [ "./librdkafka/src/CMakeLists.txt", "./librdkafka/src/Makefile", + "./librdkafka/src/README.lz4.md", "./librdkafka/src/generate_proto.sh", "./librdkafka/src/librdkafka_cgrp_synch.png", - "./librdkafka/src/statistics_schema.json", + "./librdkafka/src/opentelemetry/metrics.options", "./librdkafka/src/rdkafka_sasl_win32.c", "./librdkafka/src/rdwin32.h", + "./librdkafka/src/statistics_schema.json", "./librdkafka/src/win32_config.h", // Remove dependency on cURL. Disabling `ENABLE_CURL` and `WITH_CURL` does // not appear to prevent processing of the below files, so we have to exclude @@ -52,7 +54,6 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/apple/swift-nio.git", from: "2.55.0"), - .package(url: "https://github.com/apple/swift-nio-ssl", from: "2.25.0"), .package(url: "https://github.com/swift-server/swift-service-lifecycle.git", from: "2.1.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"), .package(url: "https://github.com/apple/swift-metrics", from: "2.4.1"), @@ -64,7 +65,7 @@ let package = Package( .target( name: "Crdkafka", dependencies: [ - .product(name: "NIOSSL", package: "swift-nio-ssl"), + "COpenSSL", .product(name: "libzstd", package: "zstd"), ], exclude: rdkafkaExclude, @@ -73,7 +74,6 @@ let package = Package( cSettings: [ // dummy folder, because config.h is included as "../config.h" in librdkafka .headerSearchPath("./custom/config/dummy"), - .headerSearchPath("./custom/include"), .headerSearchPath("./librdkafka/src"), .define("_GNU_SOURCE", to: "1"), // Fix build error for Swift 5.9 onwards ], @@ -98,6 +98,14 @@ let package = Package( "Kafka" ] ), + .systemLibrary( + name: "COpenSSL", + pkgConfig: "openssl", + providers: [ + .brew(["libressl"]), + .apt(["libssl-dev"]), + ] + ), .testTarget( name: "KafkaTests", dependencies: [ @@ -113,14 +121,21 @@ let package = Package( ) for target in package.targets { - var settings = target.swiftSettings ?? [] - settings.append(.enableExperimentalFeature("StrictConcurrency=complete")) - target.swiftSettings = settings + switch target.type { + case .regular, .test, .executable: + var settings = target.swiftSettings ?? [] + settings.append(.enableExperimentalFeature("StrictConcurrency=complete")) + target.swiftSettings = settings + case .macro, .plugin, .system, .binary: + break // These targets do not support settings + @unknown default: + fatalError("Update to handle new target type \(target.type)") + } } // --- STANDARD CROSS-REPO SETTINGS DO NOT EDIT --- // for target in package.targets { - if target.type != .plugin { + if target.type != .plugin && target.type != .system { var settings = target.swiftSettings ?? [] // https://github.com/swiftlang/swift-evolution/blob/main/proposals/0444-member-import-visibility.md settings.append(.enableUpcomingFeature("MemberImportVisibility")) diff --git a/README.md b/README.md index 204c9b2f..c5dcf824 100644 --- a/README.md +++ b/README.md @@ -237,9 +237,13 @@ config.securityProtocol = .saslTLS( ## librdkafka -The Package depends on [the librdkafka library](https://github.com/confluentinc/librdkafka), which is included as a git submodule. +The Package depends on [the `librdkafka` library](https://github.com/confluentinc/librdkafka), which is included as a git submodule. It has source files that are excluded in `Package.swift`. +### Dependencies + +`librdkafka` depends on `openssl`, meaning that `libssl-dev` must be present at build time. + ## Development Setup We provide a Docker environment for this package. This will automatically start a local Kafka server and run the package tests. diff --git a/Sources/COpenSSL/module.modulemap b/Sources/COpenSSL/module.modulemap new file mode 100644 index 00000000..e69de29b diff --git a/Sources/Crdkafka/custom/include/openssl/err.h b/Sources/Crdkafka/custom/include/openssl/err.h deleted file mode 100644 index 61a232a5..00000000 --- a/Sources/Crdkafka/custom/include/openssl/err.h +++ /dev/null @@ -1,15 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_err.h" diff --git a/Sources/Crdkafka/custom/include/openssl/evp.h b/Sources/Crdkafka/custom/include/openssl/evp.h deleted file mode 100644 index 54218188..00000000 --- a/Sources/Crdkafka/custom/include/openssl/evp.h +++ /dev/null @@ -1,15 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_evp.h" diff --git a/Sources/Crdkafka/custom/include/openssl/hmac.h b/Sources/Crdkafka/custom/include/openssl/hmac.h deleted file mode 100644 index 60a44389..00000000 --- a/Sources/Crdkafka/custom/include/openssl/hmac.h +++ /dev/null @@ -1,15 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_hmac.h" diff --git a/Sources/Crdkafka/custom/include/openssl/pkcs12.h b/Sources/Crdkafka/custom/include/openssl/pkcs12.h deleted file mode 100644 index 5d13efe2..00000000 --- a/Sources/Crdkafka/custom/include/openssl/pkcs12.h +++ /dev/null @@ -1,15 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_pkcs12.h" diff --git a/Sources/Crdkafka/custom/include/openssl/sha.h b/Sources/Crdkafka/custom/include/openssl/sha.h deleted file mode 100644 index e58bcc98..00000000 --- a/Sources/Crdkafka/custom/include/openssl/sha.h +++ /dev/null @@ -1,15 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_sha.h" diff --git a/Sources/Crdkafka/custom/include/openssl/ssl.h b/Sources/Crdkafka/custom/include/openssl/ssl.h deleted file mode 100644 index c399c7d3..00000000 --- a/Sources/Crdkafka/custom/include/openssl/ssl.h +++ /dev/null @@ -1,17 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_ssl.h" - -#define RAND_priv_bytes(buf, num) RAND_bytes((buf), (num)) diff --git a/Sources/Crdkafka/custom/include/openssl/x509.h b/Sources/Crdkafka/custom/include/openssl/x509.h deleted file mode 100644 index 3db443e9..00000000 --- a/Sources/Crdkafka/custom/include/openssl/x509.h +++ /dev/null @@ -1,15 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_x509.h" diff --git a/Sources/Crdkafka/custom/include/openssl/x509_vfy.h b/Sources/Crdkafka/custom/include/openssl/x509_vfy.h deleted file mode 100644 index aba8dcb1..00000000 --- a/Sources/Crdkafka/custom/include/openssl/x509_vfy.h +++ /dev/null @@ -1,15 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the swift-kafka-client open source project -// -// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "CNIOBoringSSL_x509_vfy.h" diff --git a/Sources/Crdkafka/librdkafka b/Sources/Crdkafka/librdkafka index 267367c9..b4c60857 160000 --- a/Sources/Crdkafka/librdkafka +++ b/Sources/Crdkafka/librdkafka @@ -1 +1 @@ -Subproject commit 267367c9475c2154e72eafe6ff1957518cb2ed1a +Subproject commit b4c608570f796c18ff2211a7af876046d264d392 diff --git a/docker/Dockerfile b/docker/Dockerfile index 322781d9..760b2932 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,9 +15,9 @@ ENV LANGUAGE en_US.UTF-8 # Dependencies RUN apt-get update RUN apt-get install libsasl2-dev -y +RUN apt-get install libssl-dev -y RUN apt-get install libjemalloc-dev -y - # tools RUN mkdir -p $HOME/.tools RUN echo 'export PATH="$HOME/.tools:$PATH"' >> $HOME/.profile