Merge branch 'master' into master

Signed-off-by: Tomasz Wilk <[email protected]>

Author: tomekwilk

.github/workflows/cron-scorecards-analysis.yaml

@@ -31,7 +31,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186
+        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/pr-compile-check.yaml

@@ -33,7 +33,7 @@ jobs:
 
   # Sanity check for compilation using system libraries
   pr-compile-system-libs:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os.version }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
@@ -49,18 +49,26 @@ jobs:
           - clang:
             cc: clang
             cxx: clang++
+        os:
+          - version: ubuntu-22.04
+            clang: "clang-12"
+          - version: ubuntu-24.04
+            clang: "clang-14"
+
     steps:
-      - name: Setup environment
+      - name: Setup environment for ${{ matrix.os.version }} with ${{ matrix.os.clang }}
         run: |
           sudo apt-get update
-          sudo apt-get install -y curl gcc-9 g++-9 clang-12 libsystemd-dev gcovr libyaml-dev
+          sudo apt-get install -y curl gcc-9 g++-9 ${CLANG_PKG} libsystemd-dev gcovr libyaml-dev
           sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
+        env:
+          CLANG_PKG: ${{ matrix.os.clang }}
 
       - name: Install system libraries for this test
         run: |
           sudo apt-get update
           sudo apt-get install -y libc-ares-dev libjemalloc-dev libluajit-5.1-dev \
-            libnghttp2-dev libsqlite3-dev libzstd-dev
+            libnghttp2-dev libsqlite3-dev libzstd-dev libmsgpack-dev
           mkdir -p /tmp/libbacktrace/build && \
             curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \
             tar --strip-components=1 -xzC /tmp/libbacktrace/ && \
@@ -80,7 +88,7 @@ jobs:
           echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
           sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
-          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/${CLANG_PKG} 90
           cmake $GLOBAL_OPTS $FLB_OPT  ../
           make -j $nparallel
         working-directory: build
@@ -89,6 +97,7 @@ jobs:
           CXX: ${{ matrix.compiler.cxx }}
           FLB_OPT: ${{ matrix.flb_option }}
           GLOBAL_OPTS: "-DFLB_JEMALLOC=On -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
+          CLANG_PKG: ${{ matrix.os.clang }}
 
       - name: Display dependencies w/ ldd
         run: |
@@ -102,3 +111,11 @@ jobs:
           echo "$ldd_result" | grep libsqlite3
           echo "$ldd_result" | grep libzstd
         working-directory: build
+
+      - name: Display dependencies w/ ldd for libmsgpack
+        if: matrix.os.version == 'ubuntu-24.04'
+        run: |
+          export ldd_result=$(ldd ./bin/fluent-bit)
+          echo "ldd result:"
+          echo "$ldd_result" | grep libmsgpack
+        working-directory: build

CMakeLists.txt

@@ -8,7 +8,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  4)
 set(FLB_VERSION_MINOR  0)
-set(FLB_VERSION_PATCH  3)
+set(FLB_VERSION_PATCH  4)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -543,7 +543,11 @@ add_subdirectory(${FLB_PATH_LIB_FLUENT_OTEL} EXCLUDE_FROM_ALL)
 # MsgPack options
 if(FLB_PREFER_SYSTEM_LIB_MSGPACK)
   find_package(PkgConfig)
+  # Try msgpack first (Ubuntu/Debian), then msgpack-c (Arch Linux)
   pkg_check_modules(MSGPACK msgpack>=4.0.0)
+  if(NOT MSGPACK_FOUND)
+    pkg_check_modules(MSGPACK msgpack-c>=4.0.0)
+  endif()
 endif()
 if(MSGPACK_FOUND)
   include_directories(${MSGPACK_INCLUDE_DIRS})

dockerfiles/Dockerfile

@@ -13,7 +13,7 @@
 # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/
 
 # Set this to the current release version: it gets done so as part of the release.
-ARG RELEASE_VERSION=4.0.3
+ARG RELEASE_VERSION=4.0.4
 
 # For multi-arch builds - assumption is running on an AMD64 host
 FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32

fluent-bit-4.0.3.bb renamed to fluent-bit-4.0.4.bb

@@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93"
 SECTION = "net"
 
 PR = "r0"
-PV = "4.0.3"
+PV = "4.0.4"
 
 SRCREV = "v${PV}"
 SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1"

include/fluent-bit/flb_api.h

@@ -22,6 +22,7 @@
 
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_output.h>
+#include <fluent-bit/flb_custom.h>
 
 struct flb_api {
     const char *(*output_get_property) (const char *, struct flb_output_instance *);
@@ -33,6 +34,11 @@ struct flb_api {
     void (*log_print) (int, const char*, int, const char*, ...);
     int (*input_log_check) (struct flb_input_instance *, int);
     int (*output_log_check) (struct flb_output_instance *, int);
+
+    /* To preserve ABI, we need to add these APIs after the
+     * input/output definitions. */
+    const char *(*custom_get_property) (const char *, struct flb_custom_instance *);
+    int (*custom_log_check) (struct flb_custom_instance *, int);
 };
 
 #ifdef FLB_CORE

include/fluent-bit/flb_config.h

@@ -67,9 +67,10 @@ struct flb_config {
      * shutdown when all remaining tasks are flushed
      */
     int grace;
-    int grace_count;          /* Count of grace shutdown tries  */
-    flb_pipefd_t flush_fd;    /* Timer FD associated to flush   */
-    int convert_nan_to_null;  /* convert null to nan ?          */
+    int grace_count;          /* Count of grace shutdown tries              */
+    int grace_input;          /* Shutdown grace to keep inputs ingesting    */
+    flb_pipefd_t flush_fd;    /* Timer FD associated to flush               */
+    int convert_nan_to_null;  /* Convert null to nan ?                      */
 
     int daemon;               /* Run as a daemon ?              */
     flb_pipefd_t shutdown_fd; /* Shutdown FD, 5 seconds         */
@@ -243,8 +244,11 @@ struct flb_config {
     int   storage_max_chunks_up;    /* max number of chunks 'up' in memory */
     int   storage_del_bad_chunks;   /* delete irrecoverable chunks */
     char *storage_bl_mem_limit;     /* storage backlog memory limit */
+    int   storage_bl_flush_on_shutdown; /* enable/disable backlog chunks flush on shutdown */
     struct flb_storage_metrics *storage_metrics_ctx; /* storage metrics context */
     int   storage_trim_files;       /* enable/disable file trimming */
+    char *storage_type;             /* global storage type */
+    int   storage_inherit;          /* apply storage type to inputs */
 
     /* Embedded SQL Database support (SQLite3) */
 #ifdef FLB_HAVE_SQLDB
@@ -392,10 +396,14 @@ enum conf_type {
 #define FLB_CONF_STORAGE_METRICS       "storage.metrics"
 #define FLB_CONF_STORAGE_CHECKSUM      "storage.checksum"
 #define FLB_CONF_STORAGE_BL_MEM_LIMIT  "storage.backlog.mem_limit"
+#define FLB_CONF_STORAGE_BL_FLUSH_ON_SHUTDOWN \
+                                       "storage.backlog.flush_on_shutdown" 
 #define FLB_CONF_STORAGE_MAX_CHUNKS_UP "storage.max_chunks_up"
 #define FLB_CONF_STORAGE_DELETE_IRRECOVERABLE_CHUNKS \
                                        "storage.delete_irrecoverable_chunks"
 #define FLB_CONF_STORAGE_TRIM_FILES    "storage.trim_files"
+#define FLB_CONF_STORAGE_TYPE          "storage.type"
+#define FLB_CONF_STORAGE_INHERIT       "storage.inherit"
 
 /* Coroutines */
 #define FLB_CONF_STR_CORO_STACK_SIZE "Coro_Stack_Size"

include/fluent-bit/flb_custom.h

@@ -31,9 +31,20 @@
 #define FLB_CUSTOM_NET_CLIENT   1   /* custom may use upstream net.* properties  */
 #define FLB_CUSTOM_NET_SERVER   2   /* custom may use downstream net.* properties  */
 
+/* Custom plugin types */
+#define FLB_CUSTOM_PLUGIN_CORE   0
+#define FLB_CUSTOM_PLUGIN_PROXY  1
+
 struct flb_custom_instance;
 
 struct flb_custom_plugin {
+    /*
+     * The type defines if this is a core-based plugin or it's handled by
+     * some specific proxy.
+     */
+    int type;
+    void *proxy;
+
     int flags;             /* Flags (not available at the moment */
     char *name;            /* Custom plugin short name           */
     char *description;     /* Description                        */
@@ -49,6 +60,9 @@ struct flb_custom_plugin {
                    void *, struct flb_config *);
     int (*cb_exit) (void *, struct flb_config *);
 
+    /* Destroy */
+    void (*cb_destroy) (struct flb_custom_plugin *);
+
     struct mk_list _head;  /* Link to parent list (config->custom) */
 };
 
@@ -96,5 +110,6 @@ int flb_custom_plugin_property_check(struct flb_custom_instance *ins,
 int flb_custom_init_all(struct flb_config *config);
 void flb_custom_set_context(struct flb_custom_instance *ins, void *context);
 void flb_custom_instance_destroy(struct flb_custom_instance *ins);
+int flb_custom_log_check(struct flb_custom_instance *ins, int l);
 
 #endif

include/fluent-bit/flb_engine.h

@@ -37,6 +37,7 @@ int flb_engine_exit_status(struct flb_config *config, int status);
 int flb_engine_shutdown(struct flb_config *config);
 int flb_engine_destroy_tasks(struct mk_list *tasks);
 void flb_engine_reschedule_retries(struct flb_config *config);
+void flb_engine_stop_ingestion(struct flb_config *config);
 
 /* Engine event loop */
 void flb_engine_evl_init();

include/fluent-bit/flb_log.h

@@ -28,6 +28,8 @@
 #include <fluent-bit/flb_worker.h>
 #include <fluent-bit/flb_config.h>
 #include <fluent-bit/flb_sds.h>
+#include <cmetrics/cmetrics.h>
+#include <cmetrics/cmt_counter.h>
 #include <inttypes.h>
 #include <errno.h>
 #include <stdarg.h>
@@ -72,6 +74,7 @@ struct flb_log {
     pthread_t tid;             /* thread ID   */
     struct flb_worker *worker; /* non-real worker reference */
     struct mk_event_loop *evl;
+    struct flb_log_metrics *metrics;
 
     /* Initialization variables */
     int pth_init;
@@ -92,6 +95,14 @@ struct flb_log_cache {
     struct mk_list entries;         /* list for entries */
 };
 
+/* Global metrics for logging calls. */
+struct flb_log_metrics {
+    struct cmt *cmt;
+
+    /* cmetrics */
+    struct cmt_counter *logs_total_counter; /* total number of logs (by message type) */
+};
+
 /*
  * This function is used by plugins interface to check if an incoming log message
  * should be logged or not based in the log levels defined.