flb_iconv: charset decoding/encoding

CMakeLists.txt

@@ -73,6 +73,7 @@ option(FLB_SYSTEM_STRPTIME    "Use strptime in system libc"  Yes)
 option(FLB_STATIC_CONF        "Build binary using static configuration")
 option(FLB_STREAM_PROCESSOR   "Enable Stream Processor"      Yes)
 option(FLB_CORO_STACK_SIZE    "Set coroutine stack size")
+option(FLB_ICONV              "Enable iconv string decoding"  No)
 
 # Metrics: Experimental Feature, disabled by default on 0.12 series
 # but enabled in the upcoming 0.13 release. Note that development
@@ -174,6 +175,7 @@ if(FLB_ALL)
   # Global
   set(FLB_DEBUG        1)
   set(FLB_TLS          1)
+  set(FLB_ICONV        1)
 
   # Input plugins
   set(FLB_IN_CPU       1)
@@ -203,6 +205,7 @@ if(FLB_ALL)
   set(FLB_OUT_STDOUT   1)
   set(FLB_OUT_LIB      1)
   set(FLB_OUT_FLOWCOUNTER 1)
+
 endif()
 
 if(FLB_DEV)
@@ -649,6 +652,67 @@ if(FLB_INOTIFY)
   endif()
 endif()
 
+# iconv
+if(FLB_ICONV)
+
+  if(CMAKE_VERSION VERSION_LESS 3.11.0)
+     message("cmake older than 3.11.0 .. no FindIconv module... testing here")
+     if(NOT DEFINED Iconv_IS_BUILT_IN)
+       if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARY)
+          check_c_source_compiles(
+                "
+                #include <stddef.h>
+                #include <iconv.h>
+                int main() {
+                  char *a, *b;
+                  size_t i, j;
+                  iconv_t ic;
+                  ic = iconv_open(\"to\", \"from\");
+                  iconv(ic, &a, &i, &b, &j);
+                  iconv_close(ic);
+                }
+                " Iconv_IS_BUILT_IN)
+       endif()
+    endif()
+    if(NOT Iconv_IS_BUILT_IN)
+      if(DEFINED Iconv_INCLUDE_DIR AND NOT DEFINED Iconv_LIBRARIES)
+        set(CMAKE_REQUIRED_LIBRARIES  "iconv")
+        check_c_source_compiles(
+                "
+                #include <stddef.h>
+                #include <iconv.h>
+                int main() {
+                  char *a, *b;
+                  size_t i, j;
+                  iconv_t ic;
+                  ic = iconv_open(\"to\", \"from\");
+                  iconv(ic, &a, &i, &b, &j);
+                  iconv_close(ic);
+                }
+                " Iconv_FOUND)
+        if(Iconv_FOUND)
+	   set(Iconv_LIBRARIES "iconv")
+        endif()
+      endif()
+    endif()
+
+  else()
+    include(FindIconv)
+  endif()
+
+  if(Iconv_IS_BUILT_IN)
+    message("Iconv_IS_BUILT_IN builtin is defined")
+    FLB_DEFINITION(FLB_HAVE_ICONV)
+  elseif(Iconv_LIBRARIES)
+    message("Iconv_LIBRARIES defined '${Iconv_LIBRARIES}'")
+    FLB_DEFINITION(FLB_HAVE_ICONV)
+  else()
+    message("FLB_ICONV defined iconv not given")
+    set(FLB_ICONV NO)
+  endif()
+
+endif()
+
 configure_file(
   "${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h.in"
   "${PROJECT_SOURCE_DIR}/include/fluent-bit/flb_info.h"

include/fluent-bit/flb_iconv.h

@@ -0,0 +1,46 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019      The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_ICONV_H
+#define FLB_ICONV_H
+
+#include <iconv.h>
+
+
+#define FLB_ICONV_SUCCESS      0
+#define FLB_ICONV_NOT_CHANGED  1
+#define FLB_ICONV_FAILURE      -1
+
+#define FLB_ICONV_ACCEPT_NOT_CHANGED  0x01
+
+struct flb_iconv {
+    iconv_t conv;
+};
+
+struct flb_iconv *flb_iconv_open(char *to, char *from);
+
+int flb_iconv_execute(struct flb_iconv *ic,
+                      char *str, size_t slen,
+                      char **result, size_t *result_len,
+                      unsigned int flags);
+
+void flb_iconv_close(struct flb_iconv *ic);
+
+#endif

plugins/in_syslog/syslog.h

@@ -24,6 +24,10 @@
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_input.h>
 
+#ifndef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
+
 /* Syslog modes */
 #define FLB_SYSLOG_UNIX_TCP  1
 #define FLB_SYSLOG_UNIX_UDP  2
@@ -63,6 +67,10 @@ struct flb_syslog {
     struct mk_list connections;
     struct mk_event_loop *evl;
     struct flb_input_instance *ins;
+
+#ifdef FLB_HAVE_ICONV
+    struct flb_iconv *iconvert;
+#endif
 };
 
 #endif

plugins/in_syslog/syslog_conf.c

@@ -25,6 +25,9 @@
 #include <fluent-bit/flb_log.h>
 #include <fluent-bit/flb_parser.h>
 #include <fluent-bit/flb_utils.h>
+#ifdef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
 
 #include "syslog.h"
 #include "syslog_server.h"
@@ -36,6 +39,9 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *ins,
     const char *tmp;
     char port[16];
     struct flb_syslog *ctx;
+#ifdef FLB_HAVE_ICONV
+    char *tmp2;
+#endif
 
     ctx = flb_calloc(1, sizeof(struct flb_syslog));
     if (!ctx) {
@@ -134,6 +140,27 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *ins,
         return NULL;
     }
 
+#ifdef FLB_HAVE_ICONV
+    tmp = flb_input_get_property("from_encoding", i_ins);
+    tmp2 = flb_input_get_property("encoding", i_ins);
+    if(tmp) {
+        if(!tmp2) {
+            tmp2 = "UTF8";
+        } else if(!strcasecmp(tmp2,"default")) {
+            tmp2 = "";
+        }
+        if(!strcasecmp(tmp,"default")) {
+            tmp = "";
+        }
+        ctx->iconvert = flb_iconv_open(tmp2,tmp);
+        if(ctx->iconvert == NULL) {
+            flb_error("[in_tail] cannot init iconv: '%s'=>'%s'", tmp, tmp2);
+        }
+    } else {
+        ctx->iconvert = NULL;
+    }
+#endif
+
     return ctx;
 }
 

plugins/in_syslog/syslog_prot.c

@@ -21,6 +21,9 @@
 #include <fluent-bit/flb_input_plugin.h>
 #include <fluent-bit/flb_parser.h>
 #include <fluent-bit/flb_time.h>
+#ifdef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
 
 #include "syslog.h"
 #include "syslog_conn.h"
@@ -63,6 +66,12 @@ int syslog_prot_process(struct syslog_conn *conn)
     size_t out_size;
     struct flb_time out_time;
     struct flb_syslog *ctx = conn->ctx;
+    char *line;
+    size_t line_len;
+#ifdef FLB_HAVE_ICONV
+    char *iconv_data;
+    size_t iconv_len;
+#endif
 
     eof = conn->buf_data;
     end = conn->buf_data + conn->buf_len;
@@ -96,8 +105,24 @@ int syslog_prot_process(struct syslog_conn *conn)
             continue;
         }
 
+#ifdef FLB_HAVE_ICONV
+        iconv_data = NULL;
+        line = p;
+        line_len = len;
+        if(ctx->iconvert) {
+            ret = flb_iconv_execute(ctx->iconvert, line, line_len, &iconv_data, &iconv_len, FLB_ICONV_ACCEPT_NOT_CHANGED);
+            if(ret == FLB_ICONV_SUCCESS) {
+                line = iconv_data;
+                line_len  = iconv_len;
+            }
+        }
+#else
+        line = p;
+        line_len = len;
+#endif
+
         /* Process the string */
-        ret = flb_parser_do(ctx->parser, p, len,
+        ret = flb_parser_do(ctx->parser, line, line_len,
                             &out_buf, &out_size, &out_time);
         if (ret >= 0) {
             pack_line(ctx, &out_time, out_buf, out_size);
@@ -108,7 +133,12 @@ int syslog_prot_process(struct syslog_conn *conn)
                          ctx->parser->name);
             flb_plg_debug(ctx->ins, "unparsed log message: %.*s", len, p);
         }
-
+#ifdef FLB_HAVE_ICONV
+        if(iconv_data) {
+            flb_free(iconv_data);
+            iconv_data = NULL;
+        }
+#endif
         conn->buf_parsed += len + 1;
         end = conn->buf_data + conn->buf_len;
         eof = conn->buf_data + conn->buf_parsed;

plugins/in_tail/tail_config.c

@@ -20,7 +20,9 @@
 
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_input_plugin.h>
-
+#ifdef FLB_HAVE_ICONV
+#include <fluent-bit/flb_iconv.h>
+#endif
 #include <stdlib.h>
 #include <fcntl.h>
 
@@ -42,6 +44,10 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
     int i;
     long nsec;
     const char *tmp;
+    ssize_t bytes;
+#ifdef FLB_HAVE_ICONV
+    char *tmp2;
+#endif
     struct flb_tail_config *ctx;
 
     ctx = flb_calloc(1, sizeof(struct flb_tail_config));
@@ -237,6 +243,27 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
     }
 #endif
 
+#ifdef FLB_HAVE_ICONV
+    tmp = flb_input_get_property("from_encoding", i_ins);
+    tmp2 = flb_input_get_property("encoding", i_ins);
+    if(tmp) {
+        if(!tmp2) {
+            tmp2 = "UTF8";
+        } else if(!strcasecmp(tmp2,"default")) {
+            tmp2 = "";
+        }
+        if(!strcasecmp(tmp,"default")) {
+            tmp = "";
+        }
+        ctx->iconvert = flb_iconv_open(tmp2,tmp);
+        if(ctx->iconvert == NULL) {
+            flb_error("[in_tail] cannot init iconv: '%s'=>'%s'", tmp, tmp2);
+        }
+    } else {
+        ctx->iconvert = NULL;
+    }
+#endif
+
 #ifdef FLB_HAVE_METRICS
     flb_metrics_add(FLB_TAIL_METRIC_F_OPENED,
                     "files_opened", ctx->ins->metrics);
@@ -268,7 +295,13 @@ int flb_tail_config_destroy(struct flb_tail_config *config)
     }
 #endif
 
-#ifdef FLB_HAVE_SQLDB
+#ifdef FLB_HAVE_ICONV
+    if(config->iconvert) {
+        iconv_close(config->iconvert);
+    }
+#endif
+
+#ifdef FLB_HAVE_SQLDB                                                         
     if (config->db != NULL) {
         flb_tail_db_close(config->db);
     }

plugins/in_tail/tail_config.h

@@ -28,6 +28,10 @@
 #ifdef FLB_HAVE_REGEX
 #include <fluent-bit/flb_regex.h>
 #endif
+#ifdef FLB_HAVE_ICONV
+#include <iconv.h>
+#endif
+
 
 /* Metrics */
 #ifdef FLB_HAVE_METRICS
@@ -111,6 +115,10 @@ struct flb_tail_config {
 
     /* Plugin input instance */
     struct flb_input_instance *ins;
+
+#ifdef FLB_HAVE_ICONV
+    struct flb_iconv *iconvert;
+#endif
 };
 
 struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,