Merge branch 'master' of github.com:tanmaya-panda1/fluent-bit into feature/added-buffering-to-azure-blob

Author: tanmaya-panda1

.github/workflows/call-build-macos.yaml

@@ -76,8 +76,10 @@ jobs:
         config:
           - name: "Apple Silicon macOS runner"
             runner: macos-14
+            cmake_version: "3.31.6"
           - name: "Intel macOS runner"
             runner: macos-14-large
+            cmake_version: "3.31.6"
 
     steps:
       - name: Checkout repository
@@ -90,6 +92,11 @@ jobs:
           brew update
           brew install bison flex libyaml openssl pkgconfig || true
 
+      - name: Install cmake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version: "${{ matrix.config.cmake_version }}"
+
       - name: Build Fluent Bit packages
         run: |
           export LIBRARY_PATH=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib:$LIBRARY_PATH

.github/workflows/call-build-windows.yaml

@@ -85,16 +85,19 @@ jobs:
             openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x86-windows-static
+            cmake_version: "3.31.6"
           - name: "Windows 64bit"
             arch: x64
             openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x64-windows-static
+            cmake_version: "3.31.6"
           - name: "Windows 64bit (Arm64)"
             arch: amd64_arm64
             openssl_dir: C:\vcpkg\packages\openssl_arm64-windows-static
             cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64"
             vcpkg_triplet: arm64-windows-static
+            cmake_version: "3.31.6"
     permissions:
       contents: read
     # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures.
@@ -113,6 +116,7 @@ jobs:
           Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe
           Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe
           echo "C:\WinFlexBison" | Out-File -FilePath $env:GITHUB_PATH -Append
+          choco install cmake --version "${{ matrix.config.cmake_version }}" --force
         env:
           WINFLEXBISON: https://github.com/lexxmark/winflexbison/releases/download/v2.5.22/win_flex_bison-2.5.22.zip
         shell: pwsh

.github/workflows/call-windows-unit-tests.yaml

@@ -39,11 +39,13 @@ jobs:
             openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x86-windows-static
+            cmake_version: "3.31.6"
           - name: "Windows 64bit"
             arch: x64
             openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x64-windows-static
+            cmake_version: "3.31.6"
     permissions:
       contents: read
     # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures.
@@ -62,6 +64,7 @@ jobs:
           Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe
           Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe
           echo "C:\WinFlexBison" | Out-File -FilePath $env:GITHUB_PATH -Append
+          choco install cmake --version "${{ matrix.config.cmake_version }}" --force
         env:
           WINFLEXBISON: https://github.com/lexxmark/winflexbison/releases/download/v2.5.22/win_flex_bison-2.5.22.zip
         shell: pwsh

.github/workflows/pr-compile-check.yaml

@@ -33,13 +33,15 @@ jobs:
 
   # Sanity check for compilation using system libraries
   pr-compile-system-libs:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
         flb_option:
           - "-DFLB_PREFER_SYSTEM_LIBS=On"
+        cmake_version:
+          - "3.31.6"
         compiler:
           - gcc:
             cc: gcc
@@ -51,24 +53,29 @@ jobs:
       - name: Setup environment
         run: |
           sudo apt-get update
-          sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \
+          sudo apt-get install -y curl gcc-9 g++-9 clang-12 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \
             libnghttp2-dev libjemalloc-dev
-          sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+          sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
           mkdir -p /tmp/libbacktrace/build && \
             curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \
             tar --strip-components=1 -xzC /tmp/libbacktrace/ && \
             pushd /tmp/libbacktrace/build && ../configure && make && sudo make install && popd
 
+      - name: Install cmake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version: "${{ matrix.cmake_version }}"
+
       - name: Checkout Fluent Bit code
         uses: actions/checkout@v4
 
       - name: ${{ matrix.compiler.cc }} & ${{ matrix.compiler.cxx }} - ${{ matrix.flb_option }}
         run: |
           export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
           echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
-          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
-          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
-          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90
           cmake $GLOBAL_OPTS $FLB_OPT  ../
           make -j $nparallel
         working-directory: build

.github/workflows/unit-tests.yaml

@@ -35,7 +35,7 @@ on:
 
 jobs:
   run-ubuntu-unit-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     strategy:
       fail-fast: false
@@ -51,6 +51,8 @@ jobs:
           - "-DFLB_SANITIZE_THREAD=On"
           - "-DFLB_SIMD=On"
           - "-DFLB_SIMD=Off"
+        cmake_version:
+          - "3.31.6"
         compiler:
           - gcc:
             cc: gcc
@@ -69,8 +71,13 @@ jobs:
       - name: Setup environment
         run: |
           sudo apt-get update
-          sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common
-          sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+          sudo apt-get install -y gcc-9 g++-9 clang-12 libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common
+          sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
+
+      - name: Install cmake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version: "${{ matrix.cmake_version }}"
 
       - uses: actions/checkout@v4
 
@@ -82,9 +89,9 @@ jobs:
       - name: ${{ matrix.compiler.cc }} & ${{ matrix.compiler.cxx }} - ${{ matrix.flb_option }}
         run: |
           echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
-          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
-          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
-          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90
           sudo usermod -a -G systemd-journal $(id -un)
           sudo -E su -p $(id -un) -c "PATH=$PATH ci/scripts/run-unit-tests.sh"
         env:
@@ -106,9 +113,16 @@ jobs:
           - "-DFLB_JEMALLOC=Off"
           - "-DFLB_SANITIZE_MEMORY=On"
           - "-DFLB_SANITIZE_THREAD=On"
+        cmake_version:
+          - "3.31.6"
     permissions:
       contents: read
     steps:
+      - name: Install cmake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version: "${{ matrix.cmake_version }}"
+
       - uses: actions/checkout@v4
       - uses: actions/checkout@v4
         with:
@@ -144,15 +158,29 @@ jobs:
             unit_test_option: "-DFLB_TESTS_INTERNAL=On"
             compiler_cc: gcc
             compiler_cxx: g++
+            cmake_version: "3.31.6"
+            cmake_home: "/opt/cmake"
+
     steps:
       - name: Checkout Fluent Bit code
         uses: actions/checkout@v4
 
       - name: Setup environment
         run: |
           sudo apt-get update
-          sudo apt-get install -y gcc-9 g++-9 clang-12 cmake flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common
+          sudo apt-get install -y gcc-9 g++-9 clang-12 flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common curl tar gzip
           sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
+          sudo mkdir -p "${CMAKE_HOME}"
+          cmake_url="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz"
+          cmake_dist="$(mktemp --suffix ".tar.gz")"
+          echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_url} -> ${cmake_dist}"
+          curl -jksSL -o "${cmake_dist}" "${cmake_url}"
+          echo "Extracting CMake ${CMAKE_VERSION}: ${cmake_dist} -> ${CMAKE_HOME}"
+          sudo tar -xzf "${cmake_dist}" -C "${CMAKE_HOME}" --strip-components 1
+          rm "${cmake_dist}"
+        env:
+          CMAKE_HOME: ${{ matrix.config.cmake_home }}
+          CMAKE_VERSION: ${{ matrix.config.cmake_version }}
 
       - name: Build and test with arm runners
         run: |
@@ -169,6 +197,10 @@ jobs:
 
             echo "CC = ${{ matrix.config.compiler_cc }}, CXX = ${{ matrix.config.compiler_cxx }}, FLB_OPT = $FLB_OPT"
 
+            if [[ -n "${CMAKE_HOME}" ]]; then
+              export PATH="${CMAKE_HOME}/bin:${PATH}"
+            fi
+
             cmake ${FLB_OPT}  ../
             make -j $nparallel
             ctest -j $nparallel --build-run-dir . --output-on-failure
@@ -177,12 +209,13 @@ jobs:
           CC: ${{ matrix.config.compiler_cc }}
           CXX: ${{ matrix.config.compiler_cxx }}
           CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
+          CMAKE_HOME: ${{ matrix.config.cmake_home }}
 
   run-qemu-ubuntu-unit-tests:
     # We chain this after Linux one as there are CPU time costs for QEMU emulation
     needs:
       - run-ubuntu-unit-tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     strategy:
       fail-fast: false
@@ -206,7 +239,8 @@ jobs:
             --volume "/etc/machine-id:/etc/machine-id"
           install: |
             apt-get update
-            apt-get install -y gcc-12 g++-12 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common
+            apt-get install -y gcc-12 g++-12 libyaml-dev flex bison libssl-dev libbpf-dev linux-tools-common
+            apt-get satisfy -y cmake "cmake (<< 4.0)"
 
             update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90
             update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90

CMakeLists.txt

@@ -174,6 +174,7 @@ option(FLB_WASM                "Build with WASM runtime support"            Yes)
 option(FLB_WAMRC               "Build with WASM AOT compiler executable"    No)
 option(FLB_WASM_STACK_PROTECT  "Build with WASM runtime with strong stack protector flags" No)
 option(FLB_ENFORCE_ALIGNMENT   "Enable limited platform specific aligned memory access" No)
+option(FLB_KAFKA               "Enable Kafka support"         Yes)
 option(FLB_ZIG                 "Enable zig integration"       Yes)
 
 # Native Metrics Support (cmetrics)
@@ -1084,16 +1085,19 @@ if(FLB_BACKTRACE)
   FLB_DEFINITION(FLB_HAVE_LIBBACKTRACE)
 endif()
 
-if(FLB_IN_KAFKA OR FLB_OUT_KAFKA)
-  if(FLB_PREFER_SYSTEM_LIB_KAFKA)
-    find_package(PkgConfig)
-    pkg_check_modules(KAFKA rdkafka>=2.3.0)
-  endif()
-  if(KAFKA_FOUND)
-    include_directories(${KAFKA_INCLUDE_DIRS})
-    link_directories(${KAFKA_LIBRARY_DIRS})
-  else()
-    include(cmake/kafka.cmake)
+# Kafka support (library in general)
+if(FLB_KAFKA)
+  if(FLB_IN_KAFKA OR FLB_OUT_KAFKA)
+    if(FLB_PREFER_SYSTEM_LIB_KAFKA)
+      find_package(PkgConfig)
+      pkg_check_modules(KAFKA rdkafka>=2.3.0)
+    endif()
+    if(KAFKA_FOUND)
+      include_directories(${KAFKA_INCLUDE_DIRS})
+      link_directories(${KAFKA_LIBRARY_DIRS})
+    else()
+      include(cmake/kafka.cmake)
+    endif()
   endif()
 endif()
 

cmake/libraries.cmake

@@ -8,7 +8,7 @@ set(FLB_PATH_LIB_CPROFILES   "lib/cprofiles")
 set(FLB_PATH_LIB_CO          "lib/flb_libco")
 set(FLB_PATH_LIB_RBTREE      "lib/rbtree")
 set(FLB_PATH_LIB_MSGPACK     "lib/msgpack-c")
-set(FLB_PATH_LIB_NGHTTP2     "lib/nghttp2")
+set(FLB_PATH_LIB_NGHTTP2     "lib/nghttp2-1.65.0")
 set(FLB_PATH_LIB_AVRO        "lib/avro")
 set(FLB_PATH_LIB_CHUNKIO     "lib/chunkio")
 set(FLB_PATH_LIB_LUAJIT      "lib/luajit-04dca791")
@@ -22,7 +22,7 @@ set(FLB_PATH_LIB_MINIZ       "lib/miniz")
 set(FLB_PATH_LIB_TUTF8E      "lib/tutf8e")
 set(FLB_PATH_LIB_CARES       "lib/c-ares-1.34.4")
 set(FLB_PATH_LIB_SNAPPY      "lib/snappy-fef67ac")
-set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.4.0")
+set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.8.0")
 set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb")
 set(FLB_PATH_LIB_WASM_MICRO_RUNTIME  "lib/wasm-micro-runtime-WAMR-1.3.3")
 set(FLB_PATH_LIB_ZSTD        "lib/zstd-1.5.7")

cmake/nghttp2.cmake

@@ -2,6 +2,8 @@
 FLB_OPTION(ENABLE_LIB_ONLY   ON)
 FLB_OPTION(ENABLE_STATIC_LIB ON)
 FLB_OPTION(ENABLE_SHARED_LIB OFF)
+FLB_OPTION(BUILD_SHARED_LIBS OFF)
+FLB_OPTION(BUILD_STATIC_LIBS ON)
 FLB_DEFINITION(NGHTTP2_STATICLIB)
 
 add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL)

dockerfiles/Dockerfile

@@ -41,7 +41,6 @@ RUN echo "deb http://deb.debian.org/debian bookworm-backports main" >> /etc/apt/
     build-essential \
     curl \
     ca-certificates \
-    cmake \
     git \
     make \
     tar \
@@ -56,6 +55,7 @@ RUN echo "deb http://deb.debian.org/debian bookworm-backports main" >> /etc/apt/
     flex \
     bison \
     libyaml-dev \
+    && apt-get satisfy -y cmake "cmake (<< 4.0)" \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -254,8 +254,9 @@ RUN echo "deb http://deb.debian.org/debian bookworm-backports main" >> /etc/apt/
     net-tools mtr netcat-openbsd bridge-utils iperf ngrep \
     openssl \
     htop atop strace iotop sysstat ncdu logrotate hdparm pciutils psmisc tree pv \
-    cmake make tar flex bison \
+    make tar flex bison \
     libssl-dev libsasl2-dev libsystemd-dev/bookworm-backports zlib1g-dev libpq-dev libyaml-dev postgresql-server-dev-all \
+    && apt-get satisfy -y cmake "cmake (<< 4.0)" \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

dockerfiles/Dockerfile.centos7

@@ -7,21 +7,32 @@ FROM centos:7
 RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \
     sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \
     yum -y update && \
-    yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
+    yum install -y rpm-build curl ca-certificates gcc gcc-c++ make bash \
     wget unzip systemd-devel wget flex bison \
-    cyrus-sasl-lib cyrus-sasl-devel openssl openss-libs openssl-devel libcurl-devel \
-    postgresql-libs postgresql-devel postgresql-server postgresql libyaml-devel && \
-    yum install -y epel-release && \
-    yum install -y cmake3
+    cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel \
+    postgresql-libs postgresql-devel postgresql-server postgresql libyaml-devel \
+    tar gzip
+
+ENV CMAKE_HOME="/opt/cmake"
+
+ARG CMAKE_VERSION="3.31.6"
+ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download"
+
+RUN mkdir -p "${CMAKE_HOME}" && \
+    cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \
+    echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \
+    curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1
+
+ENV PATH="${CMAKE_HOME}/bin:${PATH}"
 
 COPY . /src/
 WORKDIR /src/build
 
-RUN cmake3 -DCMAKE_INSTALL_PREFIX=/opt/fluent-bit/ -DCMAKE_INSTALL_SYSCONFDIR=/etc/ \
+RUN cmake -DCMAKE_INSTALL_PREFIX=/opt/fluent-bit/ -DCMAKE_INSTALL_SYSCONFDIR=/etc/ \
     -DFLB_RELEASE=On -DFLB_TRACE=On \
     -DFLB_TESTS_INTERNAL=On -DFLB_TESTS_RUNTIME=On \
     -DFLB_SQLDB=On -DFLB_HTTP_SERVER=On \
-    -DFLB_OUT_KAFKA=On \
+    -DFLB_KAFKA=off \
     -DFLB_JEMALLOC=On \
     -DFLB_CHUNK_TRACE=On \
     -DFLB_OUT_PGSQL=On \