filter-modify: Proof-of-concept integration of UTF8 encoding for string values

Signed-off-by: Nigel Stewart <[email protected]>

Author: nigels-com

include/fluent-bit/flb_encode.h

@@ -0,0 +1,28 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019      The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_ENCODE_H
+#define FLB_ENCODE_H
+
+#include <msgpack.h>
+
+void flb_msgpack_iso_8859_2_as_utf8(msgpack_packer* pk, const void* b, size_t l);
+
+#endif /* FLB_ENCODE_H */

plugins/in_tail/tail_file.c

@@ -27,6 +27,7 @@
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_input.h>
 #include <fluent-bit/flb_parser.h>
+#include <fluent-bit/flb_encode.h>
 #ifdef FLB_HAVE_REGEX
 #include <fluent-bit/flb_regex.h>
 #include <fluent-bit/flb_hash.h>
@@ -207,8 +208,7 @@ int flb_tail_file_pack_line(msgpack_sbuffer *mp_sbuf, msgpack_packer *mp_pck,
 
     msgpack_pack_str(mp_pck, ctx->key_len);
     msgpack_pack_str_body(mp_pck, ctx->key, ctx->key_len);
-    msgpack_pack_str(mp_pck, data_size);
-    msgpack_pack_str_body(mp_pck, data, data_size);
+    flb_msgpack_iso_8859_2_as_utf8(mp_pck, data, data_size);
 
     return 0;
 }

plugins/in_tail/tail_multiline.c

@@ -20,6 +20,7 @@
 
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_config.h>
+#include <fluent-bit/flb_encode.h>
 #include <fluent-bit/flb_kv.h>
 
 #include "tail_config.h"
@@ -238,8 +239,7 @@ static inline void flb_tail_mult_append_raw(char *buf, int size,
                                             struct flb_tail_config *config)
 {
     /* Append the raw string */
-    msgpack_pack_str(&file->mult_pck, size);
-    msgpack_pack_str_body(&file->mult_pck, buf, size);
+    flb_msgpack_iso_8859_2_as_utf8(&file->mult_pck, buf, size);
 }
 
 /* Check if the last key value type of a map is string or not */

src/flb_encode.c

@@ -0,0 +1,67 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019      The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_encode.h>
+#include <fluent-bit/flb_mem.h>
+
+#ifdef FLB_HAVE_ENCODE
+#include <tutf8e.h>
+#endif
+
+const size_t TUTF8_DEFAULT_BUFFER = 256;
+
+void flb_msgpack_iso_8859_2_as_utf8(msgpack_packer* pk, const void* b, size_t l)
+{
+#ifdef FLB_HAVE_ENCODE
+    size_t size = 0;
+    if (!tutf8e_buffer_length_iso_8859_2(b, l, &size) && size)
+    {
+        /* Already UTF8 encoded? */
+        if (size == l) {
+        }
+        /* Small enough for encoding to stack? */
+        else if (size<=TUTF8_DEFAULT_BUFFER)
+        {
+            size = TUTF8_DEFAULT_BUFFER;
+            char buffer[TUTF8_DEFAULT_BUFFER];
+            if (!tutf8e_buffer_encode_iso_8859_2(buffer, &size, b, l) && size) {
+                msgpack_pack_str(pk, size);
+                msgpack_pack_str_body(pk, buffer, size);
+                return;
+            }
+        }
+        /* malloc/free the encoded copy */
+        else {
+            char *buffer = (char *) flb_malloc(size);
+            if (buffer && !tutf8e_buffer_encode_iso_8859_2(buffer, &size, b, l) && size) {
+                msgpack_pack_str(pk, size);
+                msgpack_pack_str_body(pk, buffer, size);
+                free(buffer);
+                return;
+            }                        
+            free(buffer);
+        }
+    }
+#endif
+
+    /* Could not or need not encode to UTF8 */
+    msgpack_pack_str(pk, l);
+    msgpack_pack_str_body(pk, b, l);
+}