fix: add missing dependencies for SASL support on all targets

.github/workflows/call-build-macos-packages.yaml

@@ -44,7 +44,7 @@ jobs:
               run: |
                   brew update
                   # We ignore errors here as some packages may already be installed
-                  brew install bison flex gnu-sed libyaml openssl pkgconfig libgit2 cyrus-sasl rocksdb || true
+                  brew install bison flex gnu-sed libyaml openssl pkgconfig libgit2 cyrus-sasl rocksdb libtool || true
                   # Remove dynamic linkage for libyaml
                   rm -fv $(brew --prefix libyaml)/lib/*.dylib
                   # Remove dynamic linkage for zstd added by curl install

.github/workflows/call-build-windows-packages.yaml

@@ -115,6 +115,12 @@ jobs:
           C:\vcpkg\vcpkg install --recurse libgit2 --triplet ${{ matrix.config.vcpkg_triplet }}
         shell: cmd
 
+      # Unfortunately static builds are not supported and trigger link errors
+      # - name: Build libsasl with vcpkg
+      #   run: |
+      #     C:\vcpkg\vcpkg install --recurse cyrus-sasl --allow-unsupported --triplet ${{ matrix.config.vcpkg_triplet }}
+      #   shell: cmd
+
       - name: Upgrade any outdated vcpkg packages
         run: |
           C:\vcpkg\vcpkg upgrade --no-dry-run

build.sh

@@ -0,0 +1 @@
+source/packaging/build.sh

scripts/add-ignored-files.sh

@@ -15,7 +15,9 @@ while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
 SOURCE_DIR=${SOURCE_DIR:-$SCRIPT_DIR/../source}
-export FLUENT_BIT_VERSION=${FLUENT_BIT_VERSION:-v4.0.5}
+
+# Retrieve the version to use if not specified
+export FLUENT_BIT_VERSION=${FLUENT_BIT_VERSION:-$(cat "${SOURCE_DIR}"/oss_version.txt)}
 
 while IFS= read -r ignoredFile
 do

scripts/create-sync-pr.sh

@@ -7,8 +7,19 @@ set -euo pipefail
 # This script creates a GitHub PR for upstream sync commits
 # =============================================================================
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_ROOT="${SCRIPT_DIR}/.."
+# This does not work with a symlink to this script
+# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# See https://stackoverflow.com/a/246128/24637657
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+  [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE
+done
+SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+
+REPO_ROOT=${REPO_ROOT:-${SCRIPT_DIR}/..}
 cd "$REPO_ROOT"
 
 # Colors for output

scripts/sync-upstream-patches.sh

@@ -11,8 +11,19 @@ set -euo pipefail
 # we use patch files to apply changes selectively.
 # =============================================================================
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_ROOT="${SCRIPT_DIR}/.."
+# This does not work with a symlink to this script
+# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# See https://stackoverflow.com/a/246128/24637657
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+  [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE
+done
+SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+
+REPO_ROOT=${REPO_ROOT:-${SCRIPT_DIR}/..}
 cd "$REPO_ROOT" || exit 1
 
 # Configuration

source/CMakeLists.txt

@@ -193,6 +193,8 @@ option(FLB_CONFIG_YAML         "Enable YAML config format"    Yes)
 # List of plugins available and defaults for each option
 include(cmake/plugins_options.cmake)
 
+# Add any special Enterprise support
+include(cmake/enterprise.cmake)
 
 # Event loop backend (advanced users only)
 # ----------------------------------------
@@ -777,12 +779,15 @@ if(FLB_AWS AND FLB_KAFKA)
     set(FLB_HAVE_AWS_MSK_IAM ON)  # Set CMake variable
     FLB_DEFINITION(FLB_HAVE_AWS_MSK_IAM)  # Set preprocessor definition
     message(STATUS "AWS MSK IAM authentication: ENABLED")
-  else()
+  elseif(FLB_SYSTEM_WINDOWS)
+    # Windows has an issue with providing a static build of cyrus-sasl
     message(STATUS "AWS MSK IAM authentication: DISABLED (requires Kafka with libsasl2)")
+  else()
+    message(FATAL_ERROR "AWS MSK IAM authentication: DISABLED (requires Kafka with libsasl2)")
   endif()
 else()
   if(FLB_AWS AND NOT FLB_KAFKA)
-    message(STATUS "AWS MSK IAM authentication: DISABLED (requires FLB_KAFKA=ON)")
+    message(FATAL_ERROR "AWS MSK IAM authentication: DISABLED (requires FLB_KAFKA=ON)")
   endif()
 endif()
 
@@ -1589,3 +1594,5 @@ message(STATUS "Libraries install to: ${CMAKE_INSTALL_LIBDIR}")
 file(GLOB_RECURSE INSTALL_FILES "${CMAKE_INSTALL_PREFIX}/*")
 message(STATUS "Files to package: ${INSTALL_FILES}")
 
+# Verify all required options are set
+validate_required_options(FLB_HAVE_LIBYAML)

source/cmake/enterprise.cmake

@@ -5,3 +5,16 @@ FLB_DEFINITION(FLB_ENTERPRISE)
 if(CMAKE_INSTALL_PREFIX MATCHES "/opt/td-agent-bit")
   set(FLB_TD ON)
 endif()
+
+# Ensure we have specific options enabled (they may get disabled implicitly due to missing dependencies)
+function(validate_required_options)
+    set(REQUIRED_OPTIONS ${ARGV})
+
+    foreach(OPT ${REQUIRED_OPTIONS})
+        if(NOT ${OPT})
+            message(FATAL_ERROR "ERROR: ${OPT} is required but disabled.")
+        endif()
+    endforeach()
+
+    message(STATUS "All required options validated successfully")
+endfunction()

source/cmake/kafka.cmake

@@ -14,17 +14,20 @@ if(PkgConfig_FOUND)
   if(SASL_FOUND)
     message(STATUS "Found libsasl2: ${SASL_VERSION}")
     set(FLB_SASL_ENABLED ON)
-  else()
-    message(WARNING "libsasl2 not found - SASL authentication will be disabled")
   endif()
 else()
   message(WARNING "pkg-config not available - trying fallback SASL detection")
-  # Fallback detection
+endif()
+
+# Fallback when pkgconfig is not available or not working properly (centos 6)
+if(NOT FLB_SASL_ENABLED)
   find_library(SASL2_LIB NAMES sasl2)
   find_path(SASL2_INCLUDE NAMES sasl/sasl.h)
   if(SASL2_LIB AND SASL2_INCLUDE)
     set(FLB_SASL_ENABLED ON)
     message(STATUS "Found libsasl2 via fallback: ${SASL2_LIB}")
+  else()
+    message(WARNING "libsasl2 not found - SASL authentication will be disabled")
   endif()
 endif()
 

source/packaging/build.sh

@@ -1,13 +1,22 @@
 #!/bin/bash
 # Build a specific Linux target using the local source code via a container image
-set -eux
-
-# Never rely on PWD so we can invoke from anywhere
-SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+set -euo pipefail
+
+# This does not work with a symlink to this script
+# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# See https://stackoverflow.com/a/246128/24637657
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+  [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE
+done
+SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
 
 # Allow us to specify in the caller or pass variables
 FLB_DISTRO=${FLB_DISTRO:-}
-FLB_OUT_DIR=${FLB_OUT_DIR:-}
+FLB_OUT_DIR=${FLB_OUT_DIR:-agent}
 FLB_NIGHTLY_BUILD=${FLB_NIGHTLY_BUILD:-}
 DOCKER=${FLB_DOCKER_CLI:-docker}
 CACHE_ID=${CACHE_ID:-main}

source/packaging/distros/almalinux/Dockerfile

@@ -23,7 +23,7 @@ ARG CACHE_ID
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-8-${CACHE_ID} yum -y update && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -42,7 +42,7 @@ ARG CACHE_ID
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-8-arm64-${CACHE_ID} yum -y update && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -62,7 +62,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-9-${CAC
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -81,7 +81,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-9-arm64
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -101,7 +101,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-10-${CA
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all
@@ -120,7 +120,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-10-arm6
     yum install -y epel-release && \
     yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config
 
 # Split into separate layer to allow us to cache above
 RUN yum clean all

source/packaging/distros/centos/6/Dockerfile

@@ -27,7 +27,7 @@ ARG CACHE_ID
 # hadolint ignore=DL3032, DL3033
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-6-${CACHE_ID} yum -y update && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ make wget tar \
-    openssl openss-libs openssl-devel cyrus-sasl-devel \
+    openssl openss-libs openssl-devel cyrus-sasl-lib cyrus-sasl-devel \
     libyaml-devel \
     perl-core libtemplate-perl zlib-devel \
     libicu-devel scl-utils-build devtoolset-9 rh-python35 && \

source/packaging/distros/centos/7/Dockerfile

@@ -0,0 +1,113 @@
+ARG BASE_BUILDER=centos-7-base
+ARG CMAKE_VERSION=3.31.6
+# Used to differentiate in CI from main/PR builds
+ARG CACHE_ID=main
+# Multiarch support
+FROM multiarch/qemu-user-static:x86_64-aarch64 AS multiarch-aarch64
+
+# centos/7 base image
+FROM centos:7 AS centos-7-base
+
+ENV CMAKE_HOME="/opt/cmake"
+ARG CMAKE_VERSION
+ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download"
+
+COPY packaging/distros/centos/7/*.repo /etc/yum.repos.d/
+
+# Enable vault repos now main are EOL
+RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \
+    sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo
+
+ARG CACHE_ID
+# hadolint ignore=DL3033,DL3032
+RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-7-${CACHE_ID} \
+    yum -y update && \
+    yum install -y rpm-build curl ca-certificates gcc gcc-c++ make bash \
+    wget unzip systemd-devel wget flex bison \
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \
+    libyaml-devel libgit2-devel \
+    tar gzip && \
+    yum install -y epel-release && \
+    yum install -y cmake3 && \
+    yum install -y scl-utils-build devtoolset-9
+# Split into separate layer to allow us to cache above
+RUN yum clean all && scl enable devtoolset-9 bash && echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
+SHELL ["/bin/bash", "--login", "-c"]
+
+# hadolint ignore=DL4006
+RUN mkdir -p "${CMAKE_HOME}" && \
+    cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \
+    echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \
+    curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1
+
+ENV PATH="${CMAKE_HOME}/bin:${PATH}"
+
+# centos/7.arm64v8 base image
+FROM arm64v8/centos:7 AS centos-7.arm64v8-base
+
+COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+
+ENV CMAKE_HOME="/opt/cmake"
+ARG CMAKE_VERSION
+ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download"
+
+# Install DevToolset 9 as it is required for RocksDB to use C++17
+COPY packaging/distros/centos/7/*.repo /etc/yum.repos.d/
+
+# Enable vault repos now main are EOL
+RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \
+    sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo
+
+ARG CACHE_ID
+# hadolint ignore=DL3033,DL3032
+RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-7-arm64-${CACHE_ID} \
+    yum install -y rpm-build curl ca-certificates gcc gcc-c++ make bash \
+    wget unzip systemd-devel wget flex bison \
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \
+    libyaml-devel libgit2-devel \
+    tar gzip && \
+    yum install -y epel-release && \
+    yum install -y cmake3 && \
+    yum install -y scl-utils-build devtoolset-9
+# Split into separate layer to allow us to cache above
+RUN yum clean all && scl enable devtoolset-9 bash && echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
+SHELL ["/bin/bash", "--login", "-c"]
+
+# hadolint ignore=DL4006
+RUN mkdir -p "${CMAKE_HOME}" && \
+    cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \
+    echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \
+    curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1
+
+ENV PATH="${CMAKE_HOME}/bin:${PATH}"
+
+# Need larger page size
+ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
+ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
+
+# Common build for all distributions now
+# hadolint ignore=DL3006
+FROM $BASE_BUILDER AS builder
+
+ARG FLB_NIGHTLY_BUILD
+ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD
+
+# Docker context must be the base of the repo
+WORKDIR /tmp/fluent-bit/
+COPY . ./
+
+WORKDIR /tmp/fluent-bit/build/
+# CMake configuration variables
+ARG CMAKE_INSTALL_PREFIX=/opt/fluentdo-agent/
+ARG CMAKE_INSTALL_SYSCONFDIR=/etc/
+
+RUN PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH \
+    cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
+    -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \
+    -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \
+    -DFLB_JEMALLOC_OPTIONS="$FLB_JEMALLOC_OPTIONS" \
+    ../
+
+VOLUME [ "/output" ]
+# Ensure we run a login shell to trigger the devtoolset enabling
+CMD [ "/bin/bash", "-l", "-c", "scl enable devtoolset-9 bash && make --no-print-directory -j 4 && cpack -G RPM && cp *.rpm /output/" ]

source/packaging/distros/centos/7/scl.repo

@@ -0,0 +1,12 @@
+# Used for devtoolset installation
+[centos-sclo-rh]
+name=CentOS-7 - SCLo rh
+baseurl=http://vault.centos.org/centos/7/sclo/$basearch/rh/
+gpgcheck=0
+enabled=1
+
+[centos-sclo-sclo]
+name=CentOS-7 - SCLo sclo
+baseurl=http://vault.centos.org/centos/7/sclo/$basearch/sclo/
+gpgcheck=0
+enabled=1