in_tail: tests: Add UTF-16LE and UTF-16BE with BOM test cases for unicode encoder testing

Signed-off-by: Hiroshi Hatake <[email protected]>

Author: cosmo0920

tests/runtime/data/tail/log/unicode_be_c.log

@@ -0,0 +1 @@
+{"log":"在 Fluent Bit 中处理汉字，感觉像是在做梦😀。"}

tests/runtime/data/tail/log/unicode_be_j.log

@@ -0,0 +1 @@
+{"log":"にほんごテストログふぁいる。文字エンコーディングをUnicodeにできる！？☕😀⚪⚫🔴🔵🟠🟡🟢🟣🟤🇺🇸🇯🇵"}

tests/runtime/data/tail/log/unicode_c.log

@@ -0,0 +1 @@
+{"log":"用汉字在 Fluent Bit 中处理日志，就像是一个梦一样😀"}

tests/runtime/data/tail/log/unicode_j.log

@@ -0,0 +1 @@
+{"log":"にほんごテストログふぁいる。文字エンコーディングをUnicodeにできる！？☕😀⚪⚫🔴🔵🟠🟡🟢🟣🟤🇺🇸🇯🇵"}

tests/runtime/data/tail/out/unicode_be_c.out

@@ -26,6 +26,9 @@ Approach for this tests is basing on filter_kubernetes tests
 #include <fluent-bit/flb_time.h>
 #include <fluent-bit/flb_pthread.h>
 #include <fluent-bit/flb_compat.h>
+#ifdef FLB_HAVE_UNICODE_ENCODER
+#include <fluent-bit/simdutf/flb_simdutf_connector.h>
+#endif
 #include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -436,6 +439,52 @@ static int cb_check_result(void *record, size_t size, void *data)
     return 0;
 }
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+static int cb_check_result_unicode(void *record, size_t size, void *data)
+{
+    struct tail_test_result *result;
+    struct tail_file_lines *out;
+    int valid = FLB_FALSE;
+
+    result = (struct tail_test_result *) data;
+
+    char *check;
+
+    out = get_out_file_content(result->target);
+    if (!out->lines_c) {
+        goto exit;
+    }
+
+    valid = flb_simdutf_connector_validate_utf8(record, size);
+    if (valid == FLB_FALSE) {
+        goto exit;
+    }
+    /*
+      * Our validation is: check that the one of the output lines
+      * in the output record.
+      */
+    int i;
+    result->nLines = out->lines_c;
+    for (i=0; i<out->lines_c; i++) {
+      check = strstr(record, out->lines[i]);
+      if (check != NULL) {
+          result->nMatched++;
+          goto exit;
+      }
+    }
+    result->nNotMatched++;
+exit:
+    if (size > 0) {
+        flb_free(record);
+    }
+    if (out->lines_c) {
+        flb_free(out->lines[0]);
+        flb_free(out);
+    }
+    return 0;
+}
+#endif
+
 void do_test(char *system, const char *target, int tExpected, int nExpected, ...)
 {
     int64_t ret;
@@ -557,6 +606,121 @@ void flb_test_in_tail_dockermode_firstline_detection()
             NULL);
 }
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+void do_test_unicode(char *system, const char *target, int nExpected, ...)
+{
+    int64_t ret;
+    flb_ctx_t    *ctx    = NULL;
+    int in_ffd;
+    int out_ffd;
+    va_list va;
+    char *key;
+    char *value;
+    char path[PATH_MAX];
+    struct tail_test_result result = {0};
+
+    result.nMatched = 0;
+    result.target = target;
+
+    struct flb_lib_out_cb cb;
+    cb.cb   = cb_check_result_unicode;
+    cb.data = &result;
+
+    /* initialize */
+    set_result(0);
+
+    ctx = flb_create();
+
+    ret = flb_service_set(ctx,
+                          "Log_Level", "error",
+                          NULL);
+    TEST_CHECK_(ret == 0, "setting service options");
+
+    in_ffd = flb_input(ctx, (char *) system, NULL);
+    TEST_CHECK(in_ffd >= 0);
+    TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0);
+
+    /* Compose path based on target */
+    snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target);
+    TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path);
+
+    TEST_CHECK(flb_input_set(ctx, in_ffd,
+                             "path"          , path,
+                             "read_from_head", "true",
+                             NULL) == 0);
+
+    va_start(va, nExpected);
+    while ((key = va_arg(va, char *))) {
+        value = va_arg(va, char *);
+        TEST_CHECK(value != NULL);
+        TEST_CHECK(flb_input_set(ctx, in_ffd, key, value, NULL) == 0);
+    }
+    va_end(va);
+
+    out_ffd = flb_output(ctx, (char *) "lib", &cb);
+    TEST_CHECK(out_ffd >= 0);
+    TEST_CHECK(flb_output_set(ctx, out_ffd,
+                              "match", "test",
+                              "format", "json",
+                              NULL) == 0);
+
+    TEST_CHECK(flb_service_set(ctx, "Flush", "0.5",
+                                    "Grace", "1",
+                                    NULL) == 0);
+
+    /* Start test */
+    /* Start the engine */
+    ret = flb_start(ctx);
+    TEST_CHECK_(ret == 0, "starting engine");
+
+    /* /\* Poll for up to 5 seconds or until we got a match *\/ */
+    /* for (ret = 0; result.nMatched <= nExpected; ret++) { */
+    /*     usleep(1000); */
+    /* } */
+
+    /* Wait until matching nExpected results */
+    wait_with_timeout(5000, &result, nExpected);
+
+    TEST_CHECK(result.nMatched == nExpected);
+    TEST_MSG("result.nMatched: %i\nnExpected: %i", result.nMatched, nExpected);
+
+    ret = flb_stop(ctx);
+    TEST_CHECK_(ret == 0, "stopping engine");
+
+    if (ctx) {
+        flb_destroy(ctx);
+    }
+}
+
+void flb_test_in_tail_utf16le_c()
+{
+    do_test_unicode("tail", "unicode_c", 1,
+                    "Unicode.Encoding", "auto",
+                    NULL);
+}
+
+void flb_test_in_tail_utf16be_c()
+{
+    do_test_unicode("tail", "unicode_be_c", 1,
+                    "Unicode.Encoding", "auto",
+                    NULL);
+}
+
+void flb_test_in_tail_utf16le_j()
+{
+    do_test_unicode("tail", "unicode_j", 1,
+                    "Unicode.Encoding", "auto",
+                    NULL);
+}
+
+void flb_test_in_tail_utf16be_j()
+{
+    do_test_unicode("tail", "unicode_be_j", 1,
+                    "Unicode.Encoding", "auto",
+                    NULL);
+}
+#endif
+
 int write_long_lines(int fd) {
     ssize_t ret;
     int i;
@@ -1978,6 +2142,13 @@ TEST_LIST = {
     {"db_compare_filename", flb_test_db_compare_filename},
 #endif
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+    {"utf16le_c", flb_test_in_tail_utf16le_c},
+    {"utf16be_c", flb_test_in_tail_utf16be_c},
+    {"utf16le_j", flb_test_in_tail_utf16le_j},
+    {"utf16be_j", flb_test_in_tail_utf16be_j},
+#endif
+
 #ifdef in_tail
     {"in_tail_dockermode",                          flb_test_in_tail_dockermode},
     {"in_tail_dockermode_splitted_line",            flb_test_in_tail_dockermode_splitted_line},