Merge branch 'fluent:master' into feature/added-buffering-to-azure-blob

Author: tanmaya-panda1

.github/workflows/call-build-windows.yaml

@@ -82,19 +82,16 @@ jobs:
         config:
           - name: "Windows 32bit"
             arch: x86
-            openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x86-windows-static
             cmake_version: "3.31.6"
           - name: "Windows 64bit"
             arch: x64
-            openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x64-windows-static
             cmake_version: "3.31.6"
           - name: "Windows 64bit (Arm64)"
             arch: amd64_arm64
-            openssl_dir: C:\vcpkg\packages\openssl_arm64-windows-static
             cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64"
             vcpkg_triplet: arm64-windows-static
             cmake_version: "3.31.6"
@@ -143,10 +140,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            C:\vcpkg\packages
-          key: ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg-${{ steps.get-date.outputs.date }}
+            C:\vcpkg\installed
+          key: ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg-installed-${{ steps.get-date.outputs.date }}
           restore-keys: |
-            ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg-
+            ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg-installed-
           enableCrossOsArchive: false
 
       - name: Build openssl with vcpkg
@@ -159,12 +156,17 @@ jobs:
           C:\vcpkg\vcpkg install --recurse libyaml --triplet ${{ matrix.config.vcpkg_triplet }}
         shell: cmd
 
+      - name: Upgrade any outdated vcpkg packages
+        run: |
+          C:\vcpkg\vcpkg upgrade --no-dry-run
+        shell: cmd
+
       - name: Save packages of vcpkg
         id: save-vcpkg-sources
         uses: actions/cache/save@v4
         with:
           path: |
-            C:\vcpkg\packages
+            C:\vcpkg\installed
           key: ${{ steps.cache-vcpkg-sources.outputs.cache-primary-key }}
           enableCrossOsArchive: false
 
@@ -174,7 +176,7 @@ jobs:
         # This is only supposed to be a workaround for now so can be easily removed later.
         if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }}
         run: |
-          cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD='${{ inputs.unstable }}' -DOPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ${{ matrix.config.cmake_additional_opt }} -DFLB_LIBYAML_DIR=C:\vcpkg\packages\libyaml_${{ matrix.config.vcpkg_triplet }} ../
+          cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD='${{ inputs.unstable }}' -DOPENSSL_ROOT_DIR='C:\vcpkg\installed\${{ matrix.config.vcpkg_triplet }}' ${{ matrix.config.cmake_additional_opt }} -DFLB_LIBYAML_DIR='C:\vcpkg\installed\${{ matrix.config.vcpkg_triplet }}' ../
           cmake --build .
           cpack
         working-directory: build

.github/workflows/call-windows-unit-tests.yaml

@@ -36,13 +36,11 @@ jobs:
         config:
           - name: "Windows 32bit"
             arch: x86
-            openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x86-windows-static
             cmake_version: "3.31.6"
           - name: "Windows 64bit"
             arch: x64
-            openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static
             cmake_additional_opt: ""
             vcpkg_triplet: x64-windows-static
             cmake_version: "3.31.6"
@@ -91,10 +89,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: |
-            C:\vcpkg\packages
-          key: ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg-${{ steps.get-date.outputs.date }}
+            C:\vcpkg\installed
+          key: ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg-installed-${{ steps.get-date.outputs.date }}
           restore-keys: |
-            ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg-
+            ${{ runner.os }}-${{ matrix.config.arch }}-wintest-vcpkg-installed-
           enableCrossOsArchive: false
 
       - name: Build openssl with vcpkg
@@ -107,12 +105,17 @@ jobs:
           C:\vcpkg\vcpkg install --recurse libyaml --triplet ${{ matrix.config.vcpkg_triplet }}
         shell: cmd
 
+      - name: Upgrade any outdated vcpkg packages
+        run: |
+          C:\vcpkg\vcpkg upgrade --no-dry-run
+        shell: cmd
+
       - name: Save packages of vcpkg
         id: save-vcpkg-sources
         uses: actions/cache/save@v4
         with:
           path: |
-            C:\vcpkg\packages
+            C:\vcpkg\installed
           key: ${{ steps.cache-unit-test-vcpkg-sources.outputs.cache-primary-key }}
           enableCrossOsArchive: false
 
@@ -121,9 +124,9 @@ jobs:
           cmake -G "NMake Makefiles" `
             -D FLB_TESTS_INTERNAL=On `
             -D FLB_NIGHTLY_BUILD='${{ inputs.unstable }}' `
-            -D OPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' `
+            -D OPENSSL_ROOT_DIR='C:\vcpkg\installed\${{ matrix.config.vcpkg_triplet }}' `
             ${{ matrix.config.cmake_additional_opt }} `
-            -D FLB_LIBYAML_DIR=C:\vcpkg\packages\libyaml_${{ matrix.config.vcpkg_triplet }} `
+            -D FLB_LIBYAML_DIR='C:\vcpkg\installed\${{ matrix.config.vcpkg_triplet }}' `
             -D FLB_WITHOUT_flb-rt-out_elasticsearch=On `
             -D FLB_WITHOUT_flb-rt-out_td=On `
             -D FLB_WITHOUT_flb-rt-out_forward=On `

.github/workflows/pr-compile-check.yaml

@@ -53,9 +53,14 @@ jobs:
       - name: Setup environment
         run: |
           sudo apt-get update
-          sudo apt-get install -y curl gcc-9 g++-9 clang-12 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \
-            libnghttp2-dev libjemalloc-dev
+          sudo apt-get install -y curl gcc-9 g++-9 clang-12 libsystemd-dev gcovr libyaml-dev
           sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
+
+      - name: Install system libraries for this test
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libc-ares-dev libjemalloc-dev libluajit-5.1-dev \
+            libnghttp2-dev libsqlite3-dev libzstd-dev
           mkdir -p /tmp/libbacktrace/build && \
             curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \
             tar --strip-components=1 -xzC /tmp/libbacktrace/ && \
@@ -87,5 +92,13 @@ jobs:
 
       - name: Display dependencies w/ ldd
         run: |
-          ldd ./bin/fluent-bit
+          export ldd_result=$(ldd ./bin/fluent-bit)
+          echo "ldd result:"
+          echo "$ldd_result"
+          echo "$ldd_result" | grep libcares
+          echo "$ldd_result" | grep libjemalloc
+          echo "$ldd_result" | grep libluajit
+          echo "$ldd_result" | grep libnghttp2
+          echo "$ldd_result" | grep libsqlite3
+          echo "$ldd_result" | grep libzstd
         working-directory: build

CMakeLists.txt

@@ -8,7 +8,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  4)
 set(FLB_VERSION_MINOR  0)
-set(FLB_VERSION_PATCH  2)
+set(FLB_VERSION_PATCH  3)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -67,9 +67,16 @@ if (MSVC)
   #   /wd4100 .. C4100 (unreferenced formal parameter)
   #   /wd5045 .. C5045 (Spectre mitigation)
   #
+  # Restore /OPT:REF after setting /Debug, enable /OPT:ICF for 64-bit
+  # builds per Microsoft recommended best practices.
   set(CMAKE_C_FLAGS "/DWIN32 /D_WINDOWS /DNDEBUG /O2 /Zi /wd4100 /wd4711 /wd5045")
-  set(CMAKE_EXE_LINKER_FLAGS "/Debug /INCREMENTAL:NO")
+  set(CMAKE_EXE_LINKER_FLAGS "/DEBUG /OPT:REF /INCREMENTAL:NO")
+  set(CMAKE_SHARED_LINKER_FLAGS "/DEBUG /OPT:REF /INCREMENTAL:NO")
   set(CMAKE_BUILD_TYPE None)
+  if (CMAKE_SIZEOF_VOID_P EQUAL 8)
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /OPT:ICF")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /OPT:ICF")
+  endif()
 
   # Use add_compile_options() to set /MT since Visual Studio
   # Generator does not notice /MT in CMAKE_C_FLAGS.
@@ -120,6 +127,19 @@ include(cmake/platform_feature_checks.cmake)
 set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH})
 find_package(Sanitizers)
 
+# Check for CXX support
+include(CheckLanguage)
+check_language(CXX)
+
+# Enable CXX features if CXX is available
+if(CMAKE_CXX_COMPILER)
+  message(STATUS "CXX compiler found, enable simdutf.")
+  set(FLB_USE_SIMDUTF Yes)
+else()
+  message(STATUS "CXX compiler not found, disable simdutf.")
+  set(FLB_USE_SIMDUTF No)
+endif()
+
 # Output paths
 set(FLB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
@@ -139,7 +159,7 @@ option(FLB_COVERAGE            "Build with code-coverage"      No)
 option(FLB_JEMALLOC            "Build with Jemalloc support"   No)
 option(FLB_REGEX               "Build with Regex support"     Yes)
 option(FLB_UTF8_ENCODER        "Build with UTF8 encoding support" Yes)
-option(FLB_UNICODE_ENCODER     "Build with Unicode (UTF-16LE, UTF-16BE) encoding support" Yes)
+option(FLB_UNICODE_ENCODER     "Build with Unicode (UTF-16LE, UTF-16BE) encoding support" ${FLB_USE_SIMDUTF})
 option(FLB_PARSER              "Build with Parser support"    Yes)
 option(FLB_TLS                 "Build with SSL/TLS support"   Yes)
 option(FLB_BINARY              "Build executable binary"      Yes)
@@ -236,7 +256,10 @@ option(FLB_PREFER_SYSTEM_LIB_CARES        "Prefer the libcares system library"
 option(FLB_PREFER_SYSTEM_LIB_JEMALLOC     "Prefer the libjemalloc system library"     ${FLB_PREFER_SYSTEM_LIBS})
 option(FLB_PREFER_SYSTEM_LIB_KAFKA        "Prefer the libkafka system library"        ${FLB_PREFER_SYSTEM_LIBS})
 option(FLB_PREFER_SYSTEM_LIB_LUAJIT       "Prefer the libluajit system library"       ${FLB_PREFER_SYSTEM_LIBS})
+option(FLB_PREFER_SYSTEM_LIB_MSGPACK      "Prefer the libmsgpack system library"      ${FLB_PREFER_SYSTEM_LIBS})
 option(FLB_PREFER_SYSTEM_LIB_NGHTTP2      "Prefer the libnghttp2 system library"      ${FLB_PREFER_SYSTEM_LIBS})
+option(FLB_PREFER_SYSTEM_LIB_SQLITE       "Prefer the libsqlite3 system library"      ${FLB_PREFER_SYSTEM_LIBS})
+option(FLB_PREFER_SYSTEM_LIB_ZSTD         "Prefer the libzstd system library"         ${FLB_PREFER_SYSTEM_LIBS})
 
 # Enable all features
 if(FLB_ALL)
@@ -518,11 +541,16 @@ FLB_OPTION(FLUENT_PROTO_PROFILES ON)
 add_subdirectory(${FLB_PATH_LIB_FLUENT_OTEL} EXCLUDE_FROM_ALL)
 
 # MsgPack options
-option(MSGPACK_ENABLE_CXX             OFF)
-option(MSGPACK_ENABLE_SHARED          OFF)
-option(MSGPACK_BUILD_TESTS            OFF)
-option(MSGPACK_BUILD_EXAMPLES         OFF)
-add_subdirectory(${FLB_PATH_LIB_MSGPACK} EXCLUDE_FROM_ALL)
+if(FLB_PREFER_SYSTEM_LIB_MSGPACK)
+  find_package(PkgConfig)
+  pkg_check_modules(MSGPACK msgpack>=4.0.0)
+endif()
+if(MSGPACK_FOUND)
+  include_directories(${MSGPACK_INCLUDE_DIRS})
+  link_directories(${MSGPACK_LIBRARY_DIRS})
+else()
+  include(cmake/msgpack.cmake)
+endif()
 
 # MPack
 add_definitions(-DMPACK_EXTENSIONS=1)
@@ -532,13 +560,16 @@ add_subdirectory(${FLB_PATH_LIB_MPACK} EXCLUDE_FROM_ALL)
 add_subdirectory(${FLB_PATH_LIB_MINIZ} EXCLUDE_FROM_ALL)
 
 # Zstd (zstd)
-set(ZSTD_BUILD_STATIC ON)
-set(ZSTD_BUILD_SHARED OFF)
-set(ZSTD_BUILD_COMPRESSION ON)
-set(ZSTD_BUILD_DECOMPRESSION ON)
-set(ZSTD_BUILD_DICTBUILDER OFF)
-set(ZSTD_BUILD_DEPRECATED OFF)
-add_subdirectory(${FLB_PATH_LIB_ZSTD}/build/cmake EXCLUDE_FROM_ALL)
+if(FLB_PREFER_SYSTEM_LIB_ZSTD)
+  find_package(PkgConfig)
+  pkg_check_modules(LIBZSTD libzstd>=1.4.8)
+endif()
+if(LIBZSTD_FOUND)
+  include_directories(${LIBZSTD_INCLUDE_DIRS})
+  link_directories(${LIBZSTD_LIBRARY_DIRS})
+else()
+  include(cmake/zstd.cmake)
+endif()
 
 # ring buffer library
 add_subdirectory(${FLB_PATH_LIB_RING_BUFFER} EXCLUDE_FROM_ALL)
@@ -746,8 +777,17 @@ if (FLB_SIGNV4)
 endif()
 
 if(FLB_SQLDB)
+  if(FLB_PREFER_SYSTEM_LIB_SQLITE)
+    find_package(PkgConfig)
+    pkg_check_modules(SQLITE sqlite3>=3.0.0)
+  endif()
+  if(SQLITE_FOUND)
+    include_directories(${SQLITE_INCLUDE_DIRS})
+    link_directories(${SQLITE_LIBRARY_DIRS})
+  else()
+    include(cmake/sqlite.cmake)
+  endif()
   FLB_DEFINITION(FLB_HAVE_SQLDB)
-  add_subdirectory(${FLB_PATH_LIB_SQLITE})
 endif()
 
 if(FLB_TRACE)
@@ -849,6 +889,18 @@ if(FLB_HAVE_GMTOFF)
   FLB_DEFINITION(FLB_HAVE_GMTOFF)
 endif()
 
+# tm_zone support
+check_c_source_compiles("
+  #include <time.h>
+  int main() {
+     struct tm tm;
+     tm.tm_zone = \"GMT\";
+     return 0;
+  }" FLB_HAVE_TIME_ZONE)
+if(FLB_HAVE_TIME_ZONE)
+  FLB_DEFINITION(FLB_HAVE_TIME_ZONE)
+endif()
+
 # clock_get_time() support for macOS.
 check_c_source_compiles("
   #include <mach/clock.h>
@@ -996,12 +1048,8 @@ else()
     set(AUTOCONF_HOST_OPT "--host=${GNU_HOST}")
 endif()
 
-if(CMAKE_GENERATOR MATCHES "Ninja")
-  if(FLB_SYSTEM_WINDOWS)
-    MESSAGE(FATAL_ERROR "Building with Ninja is not supported on Windows")
-  else()
-    set(EXTERNAL_BUILD_TOOL "make")
-  endif()
+if(CMAKE_GENERATOR MATCHES "Ninja" AND NOT FLB_SYSTEM_WINDOWS)
+  set(EXTERNAL_BUILD_TOOL "make")
 else()
   set(EXTERNAL_BUILD_TOOL "$(MAKE)")
 endif()

SECURITY.md

@@ -5,9 +5,9 @@
 
 | Version | Supported          |
 |---------| ------------------ |
+| 4.0.x   | :white_check_mark: |
 | 3.2.x   | :white_check_mark: |
-| 3.1.x   | :white_check_mark: |
-| < 3.1   | :x:                |
+| < 3.2   | :x:                |
 
 ## Reporting a Vulnerability
 

cmake/cares.cmake

@@ -10,6 +10,11 @@ if (FLB_SYSTEM_MACOS)
   FLB_DEFINITION(CARES_HAVE_ARPA_NAMESER_H)
 endif()
 
+include_directories(
+  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include
+  ${FLB_PATH_ROOT_BINARY_DIR}/${FLB_PATH_LIB_CARES}
+)
+
 add_subdirectory(${FLB_PATH_LIB_CARES})# EXCLUDE_FROM_ALL)
 
 set(LIBCARES_LIBRARIES "c-ares")

cmake/headers.cmake

@@ -20,7 +20,6 @@ include_directories(
 
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CO}
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RBTREE}
-  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MSGPACK}/include
 
   # Chunk I/O generate headers also in the binary path
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CHUNKIO}/include
@@ -29,7 +28,6 @@ include_directories(
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MONKEY}/include
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MONKEY}/include/monkey
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MBEDTLS}/include
-  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_SQLITE}
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MPACK}/src
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MINIZ}/
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_ONIGMO}
@@ -38,16 +36,6 @@ include_directories(
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CTRACES}/include
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CPROFILES}/include
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RING_BUFFER}/lwrb/src/include
-  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_ZSTD}/lib
-
-  # c-ares
-  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include
-  ${FLB_PATH_ROOT_BINARY_DIR}/${FLB_PATH_LIB_CARES}
-
-  # nghttp2
-  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/
-  ${FLB_PATH_ROOT_BINARY_DIR}/lib/nghttp2
-  ${FLB_PATH_ROOT_BINARY_DIR}/lib/nghttp2/lib/includes/
 
   ${FLB_PATH_ROOT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include
   ${FLB_PATH_ROOT_BINARY_DIR}/lib/cmetrics

cmake/msgpack.cmake

@@ -0,0 +1,10 @@
+# msgpack cmake
+option(MSGPACK_ENABLE_CXX             OFF)
+option(MSGPACK_ENABLE_SHARED          OFF)
+option(MSGPACK_BUILD_TESTS            OFF)
+option(MSGPACK_BUILD_EXAMPLES         OFF)
+include_directories(
+  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MSGPACK}/include
+)
+add_subdirectory(${FLB_PATH_LIB_MSGPACK} EXCLUDE_FROM_ALL)
+set(MSGPACK_LIBRARIES "msgpack-c-static")

cmake/nghttp2.cmake

@@ -6,5 +6,11 @@ FLB_OPTION(BUILD_SHARED_LIBS OFF)
 FLB_OPTION(BUILD_STATIC_LIBS ON)
 FLB_DEFINITION(NGHTTP2_STATICLIB)
 
+include_directories(
+    ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/
+    ${FLB_PATH_ROOT_BINARY_DIR}/lib/nghttp2
+    ${FLB_PATH_ROOT_BINARY_DIR}/lib/nghttp2/lib/includes/
+)
+
 add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL)
 set(NGHTTP2_LIBRARIES "nghttp2_static")