in_tail: add support for charset decoding

With "from_encoding" parameter you can define
input charset for in_tail plugin.

Requires FLB_ICONV (flb_iconv)

Signed-off-by: Jukka Pihl <[email protected]>

Author: bluebike

plugins/in_tail/tail_config.c

@@ -23,7 +23,9 @@
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_log.h>
 #include <fluent-bit/flb_input.h>
-
+#ifdef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
 #include <stdlib.h>
 #include <fcntl.h>
 
@@ -46,6 +48,9 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *i_ins,
     long nsec;
     ssize_t bytes;
     const char *tmp;
+#ifdef FLB_HAVE_ICONV
+    char *tmp2;
+#endif
     struct flb_tail_config *ctx;
 
     ctx = flb_calloc(1, sizeof(struct flb_tail_config));
@@ -321,6 +326,27 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *i_ins,
     }
 #endif
 
+#ifdef FLB_HAVE_ICONV
+    tmp = flb_input_get_property("from_encoding", i_ins);
+    tmp2 = flb_input_get_property("encoding", i_ins);
+    if(tmp) {
+        if(!tmp2) {
+            tmp2 = "UTF8";
+        } else if(!strcasecmp(tmp2,"default")) {
+            tmp2 = "";
+        }
+        if(!strcasecmp(tmp,"default")) {
+            tmp = "";
+        }
+        ctx->iconvert = flb_iconv_open(tmp2,tmp);
+        if(ctx->iconvert == NULL) {
+            flb_error("[in_tail] cannot init iconv: '%s'=>'%s'", tmp, tmp2);
+        }
+    } else {
+        ctx->iconvert = NULL;
+    }
+#endif
+
 #ifdef FLB_HAVE_METRICS
     flb_metrics_add(FLB_TAIL_METRIC_F_OPENED,
                     "files_opened", ctx->i_ins->metrics);
@@ -352,7 +378,12 @@ int flb_tail_config_destroy(struct flb_tail_config *config)
     }
 #endif
 
-#ifdef FLB_HAVE_SQLDB
+#ifdef FLB_HAVE_ICONV
+    if(config->iconvert) {
+        iconv_close(config->iconvert);
+    }
+#endif
+#ifdef FLB_HAVE_SQLDB                                             
     if (config->db != NULL) {
         flb_tail_db_close(config->db);
     }

plugins/in_tail/tail_config.h

@@ -28,6 +28,10 @@
 #ifdef FLB_HAVE_REGEX
 #include <fluent-bit/flb_regex.h>
 #endif
+#ifdef FLB_HAVE_ICONV
+#include <iconv.h>
+#endif
+
 
 /* Metrics */
 #ifdef FLB_HAVE_METRICS
@@ -112,6 +116,10 @@ struct flb_tail_config {
 
     /* Plugin input instance */
     struct flb_input_instance *i_ins;
+
+#ifdef FLB_HAVE_ICONV
+    struct flb_iconv *iconvert;
+#endif
 };
 
 struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *i_ins,

plugins/in_tail/tail_file.c

@@ -31,6 +31,9 @@
 #include <fluent-bit/flb_regex.h>
 #include <fluent-bit/flb_hash.h>
 #endif
+#ifdef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
 
 #include "tail.h"
 #include "tail_file.h"
@@ -236,6 +239,10 @@ static int process_content(struct flb_tail_file *file, off_t *bytes)
     msgpack_sbuffer *out_sbuf;
     msgpack_packer *out_pck;
     struct flb_tail_config *ctx = file->config;
+#ifdef FLB_HAVE_ICONV
+    char *iconv_data;
+    size_t iconv_len;
+#endif
 
     /* Create a temporal msgpack buffer */
     msgpack_sbuffer_init(&mp_sbuf);
@@ -278,6 +285,19 @@ static int process_content(struct flb_tail_file *file, off_t *bytes)
         line_len = len - crlf;
         repl_line = NULL;
 
+#ifdef FLB_HAVE_ICONV
+        iconv_data = NULL;
+        line = data;
+        line_len = len;
+        if(ctx->iconvert) {
+            ret = flb_iconv_execute(ctx->iconvert, data, len, &iconv_data, &iconv_len, FLB_ICONV_ACCEPT_NOT_CHANGED);
+            if(ret == FLB_ICONV_SUCCESS) {
+                line = iconv_data;
+                line_len  = iconv_len;
+            }
+        }
+#endif
+
         if (ctx->docker_mode) {
             ret = flb_tail_dmode_process_content(now, line, line_len,
                                                  &repl_line, &repl_line_len,
@@ -365,6 +385,11 @@ static int process_content(struct flb_tail_file *file, off_t *bytes)
 #endif
 
     go_next:
+#ifdef FLB_HAVE_ICONV
+        if(iconv_data) {
+            flb_free(iconv_data);
+        }
+#endif
         flb_free(repl_line);
         repl_line = NULL;
         /* Adjust counters */