flb_encode: Log warning when UTF8 encoding fails

Signed-off-by: Nigel Stewart <[email protected]>

Author: nigels-com

CMakeLists.txt

@@ -50,7 +50,7 @@ option(FLB_SMALL              "Optimise for small size"       No)
 option(FLB_COVERAGE           "Build with code-coverage"      No)
 option(FLB_JEMALLOC           "Build with Jemalloc support"   No)
 option(FLB_REGEX              "Build with Regex support"     Yes)
-option(FLB_ENCODE             "Build with encoding support"  Yes)
+option(FLB_ENCODE             "Build with UTF8 encoding support" Yes)
 option(FLB_PARSER             "Build with Parser support"    Yes)
 option(FLB_TLS                "Build with SSL/TLS support"    No)
 option(FLB_BINARY             "Build executable binary"      Yes)

cmake/headers.cmake

@@ -15,7 +15,7 @@ include_directories(
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_SQLITE}
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MPACK}/src
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MINIZ}/
-  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_ONIGMO} 
+  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_ONIGMO}
   ${CMAKE_CURRENT_BINARY_DIR}/include
   )
 

include/fluent-bit/flb_encode.h

@@ -27,14 +27,14 @@ typedef void *flb_encoder;
 
 #ifdef FLB_HAVE_ENCODE
 flb_encoder flb_get_encoder(const char *encoding);
-void flb_msgpack_encode_utf8(flb_encoder enc, msgpack_packer* pk, const void* b, size_t l);
+void flb_msgpack_encode_utf8(flb_encoder encoder, const char *module, msgpack_packer *pk, const void *b, size_t l);
 #else
 static inline flb_encoder flb_get_encoder(const char *encoding)
 {
     return NULL;
 }
 
-static inline void flb_msgpack_encode_utf8(flb_encoder enc, msgpack_packer* pk, const void* b, size_t l)
+static inline void flb_msgpack_encode_utf8(flb_encoder encoder, const char *module, msgpack_packer *pk, const void *b, size_t l)
 {
     msgpack_pack_str(pk, l);
     msgpack_pack_str_body(pk, b, l);

plugins/in_syslog/syslog_prot.c

@@ -45,7 +45,7 @@ static inline int pack_line(struct flb_syslog *ctx,
 
     msgpack_pack_array(&mp_pck, 2);
     flb_time_append_to_msgpack(time, &mp_pck, 0);
-    flb_msgpack_encode_utf8(ctx->encoding, &mp_pck, data, data_size);
+    flb_msgpack_encode_utf8(ctx->encoding, "in_syslog", &mp_pck, data, data_size);
 
     flb_input_chunk_append_raw(ctx->i_ins, NULL, 0, mp_sbuf.data, mp_sbuf.size);
     msgpack_sbuffer_destroy(&mp_sbuf);

plugins/in_tail/tail_file.c

@@ -208,7 +208,7 @@ int flb_tail_file_pack_line(msgpack_sbuffer *mp_sbuf, msgpack_packer *mp_pck,
 
     msgpack_pack_str(mp_pck, ctx->key_len);
     msgpack_pack_str_body(mp_pck, ctx->key, ctx->key_len);
-    flb_msgpack_encode_utf8(ctx->encoding, mp_pck, data, data_size);
+    flb_msgpack_encode_utf8(ctx->encoding, "in_tail", mp_pck, data, data_size);
 
     return 0;
 }

plugins/in_tail/tail_multiline.c

@@ -239,7 +239,7 @@ static inline void flb_tail_mult_append_raw(char *buf, int size,
                                             struct flb_tail_config *config)
 {
     /* Append the raw string */
-    flb_msgpack_encode_utf8(config->encoding, &file->mult_pck, buf, size);
+    flb_msgpack_encode_utf8(config->encoding, "in_tail", &file->mult_pck, buf, size);
 }
 
 /* Check if the last key value type of a map is string or not */

src/flb_encode.c

@@ -19,6 +19,7 @@
  */
 
 #include <fluent-bit/flb_encode.h>
+#include <fluent-bit/flb_log.h>
 #include <fluent-bit/flb_mem.h>
 
 #ifdef FLB_HAVE_ENCODE
@@ -31,35 +32,42 @@ flb_encoder flb_get_encoder(const char *encoding)
     return tutf8e_encoder(encoding);
 }
 
-void flb_msgpack_encode_utf8(flb_encoder enc, msgpack_packer* pk, const void* b, size_t l)
+void flb_msgpack_encode_utf8(flb_encoder encoder, const char *module, msgpack_packer *pk, const void *b, size_t l)
 {
-    if (enc) {
+    if (encoder) {
         size_t size = 0;
-        if (!tutf8e_encoder_buffer_length(enc, b, l, &size) && size) {
+        if (!tutf8e_encoder_buffer_length(encoder, b, l, &size) && size) {
             /* Already UTF8 encoded? */
             if (size == l) {
             }
             /* Small enough for encoding to stack? */
             else if (size<=TUTF8_BUFFER_SIZE) {
                 char buffer[TUTF8_BUFFER_SIZE];
-                if (!tutf8e_encoder_buffer_encode(enc, b, l, buffer, &size) && size) {
+                if (!tutf8e_encoder_buffer_encode(encoder, b, l, buffer, &size) && size) {
                     msgpack_pack_str(pk, size);
                     msgpack_pack_str_body(pk, buffer, size);
                     return;
                 }
+                /* Not expecting to get here ordinarily */
+                flb_warn("[%s] failed to encode to UTF8", module);
             }
             /* malloc/free the encoded copy */
             else {
                 char *buffer = (char *) flb_malloc(size);
-                if (buffer && !tutf8e_encoder_buffer_encode(enc, b, l, buffer, &size) && size) {
+                if (buffer && !tutf8e_encoder_buffer_encode(encoder, b, l, buffer, &size) && size) {
                     msgpack_pack_str(pk, size);
                     msgpack_pack_str_body(pk, buffer, size);
                     free(buffer);
                     return;
                 }
+                /* Not expecting to get here ordinarily */
                 free(buffer);
+                flb_warn("[%s] failed to encode to UTF8", module);
             }
         }
+        else {
+            flb_warn("[%s] failed to encode to UTF8", module);
+        }
     }
 
     /* Could not or need not encode to UTF8 */