in_syslog: enable charset decoding

Enable charset deocding with "from_encoding"/"encoding"
with syslog module.  Useful syslog is not using UTF-8
(example: latin1 / CP1252)

Requires: FLB_ICONV

Signed-off-by: Jukka Pihl <[email protected]>

Author: bluebike

plugins/in_syslog/syslog.h

@@ -24,6 +24,10 @@
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_input.h>
 
+#ifndef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
+
 /* Syslog modes */
 #define FLB_SYSLOG_UNIX_TCP  1
 #define FLB_SYSLOG_UNIX_UDP  2
@@ -62,6 +66,10 @@ struct flb_syslog {
     struct mk_list connections;
     struct mk_event_loop *evl;
     struct flb_input_instance *i_ins;
+
+#ifdef FLB_HAVE_ICONV
+    struct flb_iconv *iconvert;
+#endif
 };
 
 #endif

plugins/in_syslog/syslog_conf.c

@@ -25,6 +25,9 @@
 #include <fluent-bit/flb_log.h>
 #include <fluent-bit/flb_parser.h>
 #include <fluent-bit/flb_utils.h>
+#ifdef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
 
 #include "syslog.h"
 #include "syslog_server.h"
@@ -36,6 +39,9 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *i_ins,
     const char *tmp;
     char port[16];
     struct flb_syslog *ctx;
+#ifdef FLB_HAVE_ICONV
+    char *tmp2;
+#endif
 
     ctx = flb_calloc(1, sizeof(struct flb_syslog));
     if (!ctx) {
@@ -144,6 +150,27 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *i_ins,
         return NULL;
     }
 
+#ifdef FLB_HAVE_ICONV
+    tmp = flb_input_get_property("from_encoding", i_ins);
+    tmp2 = flb_input_get_property("encoding", i_ins);
+    if(tmp) {
+        if(!tmp2) {
+            tmp2 = "UTF8";
+        } else if(!strcasecmp(tmp2,"default")) {
+            tmp2 = "";
+        }
+        if(!strcasecmp(tmp,"default")) {
+            tmp = "";
+        }
+        ctx->iconvert = flb_iconv_open(tmp2,tmp);
+        if(ctx->iconvert == NULL) {
+            flb_error("[in_tail] cannot init iconv: '%s'=>'%s'", tmp, tmp2);
+        }
+    } else {
+        ctx->iconvert = NULL;
+    }
+#endif
+
     return ctx;
 }
 

plugins/in_syslog/syslog_prot.c

@@ -23,6 +23,9 @@
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_parser.h>
 #include <fluent-bit/flb_time.h>
+#ifdef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
 
 #include "syslog.h"
 #include "syslog_conn.h"
@@ -63,6 +66,12 @@ int syslog_prot_process(struct syslog_conn *conn)
     size_t out_size;
     struct flb_time out_time;
     struct flb_syslog *ctx = conn->ctx;
+    char *line;
+    size_t line_len;
+#ifdef FLB_HAVE_ICONV
+    char *iconv_data;
+    size_t iconv_len;
+#endif
 
     eof = p = conn->buf_data;
     end = conn->buf_data + conn->buf_len;
@@ -97,8 +106,24 @@ int syslog_prot_process(struct syslog_conn *conn)
             continue;
         }
 
+#ifdef FLB_HAVE_ICONV
+        iconv_data = NULL;
+        line = p;
+        line_len = len;
+        if(ctx->iconvert) {
+            ret = flb_iconv_execute(ctx->iconvert, line, line_len, &iconv_data, &iconv_len, FLB_ICONV_ACCEPT_NOT_CHANGED);
+            if(ret == FLB_ICONV_SUCCESS) {
+                line = iconv_data;
+                line_len  = iconv_len;
+            }
+        }
+#else
+        line = p;
+        line_len = len;
+#endif
+
         /* Process the string */
-        ret = flb_parser_do(ctx->parser, p, len,
+        ret = flb_parser_do(ctx->parser, line, line_len,
                             &out_buf, &out_size, &out_time);
         if (ret >= 0) {
             pack_line(ctx, &out_time, out_buf, out_size);
@@ -107,7 +132,12 @@ int syslog_prot_process(struct syslog_conn *conn)
         else {
             flb_warn("[in_syslog] error parsing log message");
         }
-
+#ifdef FLB_HAVE_ICONV
+        if(iconv_data) {
+            flb_free(iconv_data);
+            iconv_data = NULL;
+        }
+#endif
         conn->buf_parsed += len + 1;
         end = conn->buf_data + conn->buf_len;
         eof = p = conn->buf_data + conn->buf_parsed;