encoding: more work...

Signed-off-by: Jukka Pihl <[email protected]>

Author: bluebike

include/fluent-bit/flb_encoding.h

@@ -32,7 +32,8 @@ struct flb_encoding {
     const char *invalid;
 };
 
-struct flb_encoding *flb_encoding_open(const char *encoding);
+
+struct flb_encoding *flb_encoding_open(const char *encoding, const char *replacement);
 
 int flb_encoding_decode(struct flb_encoding *ec,
                         char *str, size_t slen,

plugins/in_syslog/syslog_conf.c

@@ -34,6 +34,9 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *ins,
                                       struct flb_config *config)
 {
     const char *tmp;
+#ifdef FLB_HAVE_UTF8_ENCODER
+    const char *tmp2;
+#endif
     char port[16];
     struct flb_syslog *ctx;
 
@@ -138,7 +141,8 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *ins,
     /* utf8 encoder */
     tmp = flb_input_get_property("encoding", ins);
     if (tmp) {
-        ctx->encoding = flb_encoding_open(tmp);
+        tmp2 = flb_input_get_property("encoding_replacement", ins);
+        ctx->encoding = flb_encoding_open(tmp,tmp2);
         if (!ctx->encoding) {
             flb_error("[in_syslog] illegal encoding: %s", tmp);
             syslog_conf_destroy(ctx);

plugins/in_tail/tail.c

@@ -626,6 +626,11 @@ static struct flb_config_map config_map[] = {
      0, FLB_FALSE, 0,
      "specify the charset encoder to decode message",
     },
+    {
+     FLB_CONFIG_MAP_STR, "encoding_replacement", NULL,
+     0, FLB_FALSE, 0,
+     "specify the charset encoder to decode message",
+    },
 #endif
 
     /* Multiline Options */

plugins/in_tail/tail_config.c

@@ -43,8 +43,10 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
     int i;
     long nsec;
     const char *tmp;
+#ifdef FLB_HAVE_UTF8_ENCODER    
+    const char *tmp2;
+#endif
     struct flb_tail_config *ctx;
-
     ctx = flb_calloc(1, sizeof(struct flb_tail_config));
     if (!ctx) {
         flb_errno();
@@ -157,7 +159,8 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
 #ifdef FLB_HAVE_UTF8_ENCODER
     tmp = flb_input_get_property("encoding", ins);
     if (tmp) {
-        ctx->encoding = flb_encoding_open(tmp);
+        tmp2 = flb_input_get_property("encoding_relacement", ins);
+        ctx->encoding = flb_encoding_open(tmp,tmp2);
         if (!ctx->encoding) {
             flb_plg_error(ctx->ins,"illegal encoding: %s", tmp);
             flb_tail_config_destroy(ctx);

src/flb_encoding.c

@@ -36,54 +36,57 @@
 
 /*
  *
- *  flb_encoding_open(encoding):
- *  iso-8859-1,...
- *  windows-1251 windows-1252, ..
+ *  flb_encoding_open(encoding,replacement):
+ *
+ *  encoding: 
+ *    iso-8859-1,...
+ *    windows-1251 windows-1252, ..
+ * 
+ *  replacement:
+ *      \R        use replacement character 0xFFD  (default)
+ *      \I        ignore/skip bad chars
+ *      \E        fail if bad chars
+ *      ...       use that as replacement char
  * 
- *  <charset>                     - fail if bad chars
- *  <charset>//IGNORE             - ignore bad chars
- *  <charset>//REPLACEMENT //R    - use unicode replacement chars for bad chars
- *  <charset>//QUESTION  //Q      - use '?' for bad chars
- *  <charset>///<str>             - use <str> for bad chars
  */
 
 
 static  unsigned char replacement_utf8[] = { 0xEF, 0xBF, 0xBD , 0 };
 
-struct flb_encoding *flb_encoding_open(const char *encoding) {
+
+static char *parse_replacement(const char *replacement) {
+
+    if (!replacement) {
+        return replacement_utf8;
+    }
+    if (!strcmp(replacement,"\\R")) {
+        return replacement_utf8;
+    }
+    if (!strcmp(replacement,"\\I")) {
+        return "";
+    }
+    if (!strcmp(replacement,"\\?")) {
+        return "?";
+    }
+    if (!strcmp(replacement,"\\E")) {
+        return NULL;
+    }
+    
+    return replacement;
+}
+
+struct flb_encoding *flb_encoding_open(const char *encoding, const char *replacement) {
     struct flb_encoding *ec;
     TUTF8encoder encoder;
     const char *invalid;
     char *opt;
-
-    if ((opt = strstr(encoding,"//")) != NULL) {
-        *opt = 0;
-        opt += 2;
-        if (*opt == '/') {
-            invalid = opt + 1;
-        }
-        else if (!strcmp(opt,"I") || !strcmp(opt,"IGNORE")) {
-            invalid = "";
-        }
-        else if (!strcmp(opt,"R") || !strcmp(opt,"REPLACEMENT")) {
-            invalid = (const char *) replacement_utf8;
-        }
-        else if (!strcmp(opt,"Q") || !strcmp(opt,"QUESTION")) {
-            invalid = "?";
-        }
-        else {
-            flb_error("[flb_encoding] unknown encodig option: %s", opt);
-            return NULL;
-        }
-    }
-    else {
-        invalid = NULL;
-    }
 
     if ((encoder = tutf8e_encoder(encoding)) == NULL) {
         flb_error("[flb_encoding] unknown encoding: %s", encoding);
         return NULL;
     }
+
+    invalid = parse_replacement(replacement);
 
     ec = flb_calloc(sizeof(struct flb_encoding),1);