filter-modify: Proof-of-concept integration of UTF8 encoding for string values

Signed-off-by: Nigel Stewart <[email protected]>

Author: nigels-com

CMakeLists.txt

@@ -309,6 +309,9 @@ add_subdirectory(${FLB_PATH_LIB_MPACK} EXCLUDE_FROM_ALL)
 # Miniz (zip)
 add_subdirectory(${FLB_PATH_LIB_MINIZ} EXCLUDE_FROM_ALL)
 
+# tutf8e
+add_subdirectory(${FLB_PATH_LIB_TUTF8E} EXCLUDE_FROM_ALL)
+
 # Chunk I/O
 FLB_OPTION(CIO_LIB_STATIC  ON)
 FLB_OPTION(CIO_LIB_SHARED  OFF)

cmake/headers.cmake

@@ -16,6 +16,7 @@ include_directories(
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MPACK}/src
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MINIZ}/
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_ONIGMO}
+  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_TUTF8E}/include
   ${CMAKE_CURRENT_BINARY_DIR}/include
   )
 

cmake/libraries.cmake

@@ -11,3 +11,4 @@ set(FLB_PATH_LIB_SQLITE  "lib/sqlite-amalgamation-3240000")
 set(FLB_PATH_LIB_ONIGMO  "lib/onigmo")
 set(FLB_PATH_LIB_MPACK   "lib/mpack-amalgamation-1.0")
 set(FLB_PATH_LIB_MINIZ   "lib/miniz")
+set(FLB_PATH_LIB_TUTF8E  "lib/tutf8e")

plugins/filter_modify/modify.c

@@ -31,6 +31,8 @@
 #include <fluent-bit/flb_regex.h>
 #include <msgpack.h>
 
+#include <tutf8e.h>
+
 #include "modify.h"
 
 static void condition_free(struct modify_condition *condition)
@@ -331,6 +333,9 @@ static int setup(struct filter_modify_ctx *ctx,
                 else if (strcasecmp(kv->key, "set") == 0) {
                     rule->ruletype = SET;
                 }
+                else if (strcasecmp(kv->key, "utf8") == 0) {
+                    rule->ruletype = UTF8;
+                }
                 else if (strcasecmp(kv->key, "copy") == 0) {
                     rule->ruletype = COPY;
                 }
@@ -1157,6 +1162,42 @@ static inline int apply_rule_SET(msgpack_packer * packer,
     return FLB_FILTER_MODIFIED;
 }
 
+static inline int apply_rule_UTF8(msgpack_packer * packer,
+                                 msgpack_object * map,
+                                 struct modify_rule *rule)
+{
+    int ret = FLB_FILTER_NOTOUCH;
+    int i;
+
+    int match_keys =
+        map_count_keys_matching_str(map, rule->key, rule->key_len);
+
+    if (match_keys == 0) {
+        flb_debug
+            ("[filter_modify] Rule %s to UTF8 %s : No keys matching %s found, not applying rule",
+             rule->val, rule->key, rule->key);
+    }
+    else {
+        msgpack_pack_map(packer, map->via.map.size);
+        for (i = 0; i < map->via.map.size; i++) {
+            msgpack_pack_object(packer, map->via.map.ptr[i].key);
+            if (map->via.map.ptr[i].val.type == MSGPACK_OBJECT_STR &&
+                kv_key_matches_str_rule_key(&map->via.map.ptr[i], rule)) {
+                size_t size = 512;
+                char buffer[512];
+                if (!tutf8e_buffer_encode_iso_8859_2(buffer, &size, map->via.map.ptr[i].val.via.str.ptr, map->via.map.ptr[i].val.via.str.size))
+                {
+                    helper_pack_string(packer, buffer, size);
+                    ret = FLB_FILTER_MODIFIED;
+                    continue;
+                }
+            }
+            msgpack_pack_object(packer, map->via.map.ptr[i].val);
+        }
+    }
+    return ret;
+}
+
 static inline int apply_rule_REMOVE(msgpack_packer * packer,
                                     msgpack_object * map,
                                     struct modify_rule *rule)
@@ -1222,6 +1263,8 @@ static inline int apply_modifying_rule(msgpack_packer * packer,
         return apply_rule_ADD(packer, map, rule);
     case SET:
         return apply_rule_SET(packer, map, rule);
+    case UTF8:
+        return apply_rule_UTF8(packer, map, rule);
     case REMOVE:
         return apply_rule_REMOVE(packer, map, rule);
     case REMOVE_WILDCARD:

plugins/filter_modify/modify.h

@@ -26,6 +26,7 @@ enum FLB_FILTER_MODIFY_RULETYPE {
   HARD_RENAME,
   ADD,
   SET,
+  UTF8,
   REMOVE,
   REMOVE_WILDCARD,
   REMOVE_REGEX,

src/CMakeLists.txt

@@ -229,6 +229,7 @@ set(FLB_DEPS
   mpack-static
   chunkio-static
   miniz
+  tutf8e
   ${FLB_PLUGINS}
   ${FLB_PROXY_PLUGINS}
   ${extra_libs}