in_tail: Handle encoding conversion engine for non UTF-16 encodings

Signed-off-by: Hiroshi Hatake <[email protected]>

Author: cosmo0920

plugins/in_tail/tail.c

@@ -826,6 +826,15 @@ static struct flb_config_map config_map[] = {
      "Currently, UTF-16LE, UTF-16BE, auto are supported.",
     },
 #endif
+    {
+     FLB_CONFIG_MAP_STR, "generic.encoding", NULL,
+     0, FLB_FALSE, 0,
+     "specify the preferred input encoding for converting to UTF-8. "
+     "Currently, the following encodings are supported: "
+     "ShiftJIS, UHC, GBK, GB18030, Big5, "
+     "Win866, Win874, "
+     "Win1250, Win1251, Win1252, Win2513, Win1254, Win1255, WIn1256",
+    },
     /* EOF */
     {0}
 };

plugins/in_tail/tail_config.c

@@ -114,6 +114,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
 #ifdef FLB_HAVE_UNICODE_ENCODER
     ctx->preferred_input_encoding = FLB_UNICODE_ENCODING_UNSPECIFIED;
 #endif
+    ctx->generic_input_encoding_type = FLB_GENERIC_UNSPECIFIED; /* Default is unspecified */
 
     /* Load the config map */
     ret = flb_input_config_map_set(ins, (void *) ctx);
@@ -222,6 +223,20 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
     }
 #endif
 
+    tmp = flb_input_get_property("generic.encoding", ins);
+    if (tmp) {
+        ret = flb_unicode_generic_select_encoding_type(tmp);
+        if (ret != FLB_GENERIC_UNSPECIFIED) {
+            ctx->generic_input_encoding_type = ret;
+            ctx->generic_input_encoding_name = tmp;
+        }
+        else {
+            flb_plg_error(ctx->ins, "invalid encoding 'generic.encoding' value %s", tmp);
+            flb_free(ctx);
+            return NULL;
+        }
+    }
+
 #ifdef FLB_HAVE_PARSER
     /* Config: multi-line support */
     if (ctx->multiline == FLB_TRUE) {

plugins/in_tail/tail_config.h

@@ -129,6 +129,8 @@ struct flb_tail_config {
 #ifdef FLB_HAVE_UNICODE_ENCODER
     int preferred_input_encoding;
 #endif
+    int generic_input_encoding_type;
+    const char *generic_input_encoding_name;
 
     /* Multiline */
     int multiline;             /* multiline enabled ?  */

plugins/in_tail/tail_file.c

@@ -485,6 +485,20 @@ static int process_content(struct flb_tail_file *file, size_t *bytes)
         }
     }
 #endif
+    if (ctx->generic_input_encoding_type != FLB_GENERIC_UNSPECIFIED) {
+        original_len = end - data;
+        decoded = NULL;
+        ret = flb_unicode_generic_convert_to_utf8(ctx->generic_input_encoding_name,
+                                                  (unsigned char*)data, (unsigned char**)&decoded,
+                                                  end - data);
+        if (ret > 0) {
+            data = decoded;
+            end  = data + strlen(decoded);
+        }
+        else {
+            flb_plg_error(ctx->ins, "encoding failed '%.*s' with status %d", end - data, data, ret);
+        }
+    }
 
     /* Skip null characters from the head (sometimes introduced by copy-truncate log rotation) */
     while (data < end && *data == '\0') {