conv: unicode: Add a search function for encoding types

This is intended to use for in_tail plugin for follow the current code
base of rules to search preferred encodings.

Signed-off-by: Hiroshi Hatake <[email protected]>

Author: cosmo0920

include/fluent-bit/flb_unicode.h

@@ -41,6 +41,7 @@ enum flb_unicode_generic_enc_type {
     FLB_GENERIC_GBK            = FLB_GBK,            /* GBK (Windows-936) */
     FLB_GENERIC_UHC            = FLB_UHC,            /* UHC (Windows-949) */
     FLB_GENERIC_GB18030        = FLB_GB18030,        /* GB18030 */
+    FLB_GENERIC_UNSPECIFIED    = FLB_CONV_ENCODING_UNSPECIFIED, /* Unspecified */
     _FLB_GENERIC_LAST_ENCODING_ /* mark only */
 };
 
@@ -77,6 +78,7 @@ int flb_unicode_convert(int preferred_encoding, const char *input, size_t length
 int flb_unicode_validate(const char *record, size_t size);
 
 int flb_unicode_generic_supported_encoding(const char *encoding_name);
+int flb_unicode_generic_select_encoding_type(const char *encoding_name);
 int flb_unicode_generic_convert_to_utf8(const char *encoding_name,
                                         const unsigned char *input, unsigned char **output, size_t length);
 int flb_unicode_generic_convert_from_utf8(const char *encoding_name,

include/fluent-bit/unicode/flb_conv.h

@@ -32,6 +32,9 @@
 #define FLB_CONV_ALLOCATION_FAILED   -2
 #define FLB_CONV_CONVERSION_FAILED   -3
 
+/* Unspecified encoding type */
+#define FLB_CONV_ENCODING_UNSPECIFIED -1
+
 struct flb_unicode_converter {
     const char *name;
     const char *aliases[FLB_CONV_MAX_ALIAS_LENGTH];
@@ -50,6 +53,7 @@ struct flb_unicode_converter {
 
 struct flb_unicode_converter *flb_conv_select_converter(const char *encoding_name);
 int flb_conv_supported_encoding(const char *encoding_name);
+int flb_conv_select_encoding_type(const char *encoding_name);
 int flb_conv_convert_to_utf8(const char *encoding_name,
                              const unsigned char *src, unsigned char **dest,
                              size_t len, bool no_error);

src/flb_unicode.c

@@ -47,6 +47,11 @@ int flb_unicode_generic_supported_encoding(const char *encoding_name)
     return flb_conv_supported_encoding(encoding_name);
 }
 
+int flb_unicode_generic_select_encoding_type(const char *encoding_name)
+{
+    return flb_conv_select_encoding_type(encoding_name);
+}
+
 int flb_unicode_generic_convert_to_utf8(const char *encoding_name,
                                         const unsigned char *input, unsigned char **output, size_t length)
 {

src/unicode/flb_conv.c

@@ -592,6 +592,18 @@ int flb_conv_supported_encoding(const char *encoding_name)
     return flb_conv_select_converter(encoding_name) != NULL;
 }
 
+int flb_conv_select_encoding_type(const char *encoding_name)
+{
+    struct flb_unicode_converter *conv;
+    conv = flb_conv_select_converter(encoding_name);
+
+    if (conv) {
+        return conv->encoding;
+    }
+
+    return FLB_CONV_ENCODING_UNSPECIFIED;
+}
+
 int flb_conv_convert_to_utf8(const char *encoding_name,
                              const unsigned char *src, unsigned char **dest,
                              size_t len, bool no_error)

tests/internal/unicode.c

@@ -50,6 +50,80 @@ void test_generic_converters_alias()
     }
 }
 
+struct test_enc_types {
+    char *name;
+    int type;
+};
+
+void test_generic_encoding_types()
+{
+    int encoding_type;
+    const char *encoding_name;
+    int i;
+    struct test_enc_types encodings[] = {
+        {"ShiftJIS", FLB_GENERIC_SJIS},
+        {"GB18030",  FLB_GENERIC_GB18030},
+        {"UHC",      FLB_GENERIC_UHC},
+        {"Big5",     FLB_GENERIC_BIG5},
+        {"Win866",   FLB_GENERIC_WIN866},
+        {"Win874",   FLB_GENERIC_WIN874},
+        {"Win1250",  FLB_GENERIC_WIN1250},
+        {"Win1251",  FLB_GENERIC_WIN1251},
+        {"Win1252",  FLB_GENERIC_WIN1252},
+        {"Win1253",  FLB_GENERIC_WIN1253},
+        {"Win1254",  FLB_GENERIC_WIN1254},
+        {"Win1255",  FLB_GENERIC_WIN1255},
+        {"Win1256",  FLB_GENERIC_WIN1256},
+        {"GBK",      FLB_GENERIC_GBK},
+        {NULL,       FLB_GENERIC_UNSPECIFIED},
+    };
+
+    for (i = 0; encodings[i].name != NULL; i++) {
+        encoding_type = flb_unicode_generic_select_encoding_type(encodings[i].name);
+        if (TEST_CHECK(encoding_type == encodings[i].type)) {
+            TEST_MSG("supported encoding type selection %d check failed with %s",
+                     encoding_type,
+                     encodings[i].name);
+        }
+    }
+
+    encoding_name = "Nonexisitent";
+    encoding_type = flb_unicode_generic_select_encoding_type(encoding_name);
+    if (!TEST_CHECK(encoding_type == FLB_GENERIC_UNSPECIFIED)) {
+        TEST_MSG("supported converter check unexpectedly succeeded with %s", encoding_name);
+        return;
+    }
+}
+
+void test_generic_alias_encoding_types()
+{
+    int encoding_type;
+    int i;
+    struct test_enc_types encodings[] = {
+        {"SJIS",   FLB_GENERIC_SJIS},
+        {"CP866",  FLB_GENERIC_WIN866},
+        {"CP874",  FLB_GENERIC_WIN874},
+        {"CP932",  FLB_GENERIC_SJIS},
+        {"CP936",  FLB_GENERIC_GBK},
+        {"CP950",  FLB_GENERIC_BIG5},
+        {"CP1250", FLB_GENERIC_WIN1250},
+        {"CP1251", FLB_GENERIC_WIN1251},
+        {"CP1252", FLB_GENERIC_WIN1252},
+        {"CP1253", FLB_GENERIC_WIN1253},
+        {"CP1254", FLB_GENERIC_WIN1254},
+        {"CP1255", FLB_GENERIC_WIN1255},
+        {"CP1256", FLB_GENERIC_WIN1256},
+        {NULL,     FLB_GENERIC_UNSPECIFIED},
+    };
+
+    for (i = 0; encodings[i].name != NULL; i++) {
+        encoding_type = flb_unicode_generic_select_encoding_type(encodings[i].name);
+        if (!TEST_CHECK(encoding_type == encodings[i].type)) {
+            TEST_MSG("supported converter check failed with %s", encodings[i].name);
+        }
+    }
+}
+
 void test_generic_conversions_sjis()
 {
     /* "こんにちは" in SJIS */
@@ -277,6 +351,8 @@ void test_all_generic_conversions()
 TEST_LIST = {
     { "generic_converters", test_generic_converters },
     { "generic_converters_alias", test_generic_converters_alias },
+    { "generic_encoding_types", test_generic_encoding_types },
+    { "generic_alias_encoding_types", test_generic_alias_encoding_types },
     { "generic_conversions_sjis", test_generic_conversions_sjis },
     { "generic_conversions_gbk", test_generic_conversions_gbk },
     { "generic_conversions_big5", test_generic_conversions_big5 },