Generate native language name list on compile-time

Author: botder

.gitignore

@@ -416,3 +416,4 @@ utils/DXFiles/
 !*.dll
 !*.exe
 utils/vswhere.exe
+*.generated.h

Client/core/CLocalization.cpp

@@ -15,6 +15,14 @@
 // TRANSLATORS: Replace with your language native name
 #define NATIVE_LANGUAGE_NAME _td("English")
 
+struct NativeLanguageName
+{
+    std::string locale;
+    std::string name;
+} g_nativeLanguageNames[] = {
+    #include "languages.generated.h"
+};
+
 CLocalization::CLocalization(const SString& strLocale, const SString& strLocalePath)
 {
     // Set log callbacks so we can record problems
@@ -95,12 +103,23 @@ CLanguage* CLocalization::GetLanguage(SString strLocale)
 SString CLocalization::GetLanguageNativeName(SString strLocale)
 {
     strLocale = ValidateLocale(strLocale);
+
+    // Try to find the native language name in our sorted compile-time array first, using binary search.
+    auto begin = g_nativeLanguageNames;
+    auto end = g_nativeLanguageNames + _countof(g_nativeLanguageNames);
+    auto iter = std::lower_bound(begin, end, strLocale, [](const NativeLanguageName& a, const std::string& b) { return a.locale < b; });
+
+    if (iter != end && iter->locale == strLocale)
+        return iter->name;
+
+    // If not found, we fall back to the loading the language file and using the name from there.
     SString strNativeName = GetLanguage(strLocale)->Translate(NATIVE_LANGUAGE_NAME);
     if (strNativeName == "English" && strLocale != "en_US")
     {
         // If native name not available, use English version
         strNativeName = GetLanguage(strLocale)->GetName();
     }
+
     return strNativeName;
 }
 

Client/core/premake5.lua

@@ -55,6 +55,10 @@ project "Client Core"
 		"PNG_SETJMP_NOT_SUPPORTED"
 	}
 
+	prebuildcommands {
+		"%[%{!wks.location}/../utils/gen_language_list.exe] %[%{!wks.location}/../Shared/data/MTA San Andreas/MTA/locale] %[languages.generated.h]"
+	}
+
 	filter "architecture:not x86"
 		flags { "ExcludeFromBuild" }
 

Shared/sdk/SharedUtil.cpp

@@ -18,3 +18,4 @@
     #include "SharedUtil.SysInfo.hpp"
 #endif
 #include "SharedUtil.Memory.hpp"
+#include "UTF8.hpp"

Shared/sdk/SharedUtil.hpp

@@ -25,3 +25,4 @@
 #include "SharedUtil.Logging.hpp"
 #include "SharedUtil.AsyncTaskScheduler.hpp"
 #include "SharedUtil.Memory.hpp"
+#include "UTF8.hpp"

Shared/sdk/UTF8.h

@@ -23,255 +23,26 @@
  */
 #pragma once
 
-#include "SharedUtil.IntTypes.h"
 #include <string>
 
-/* Return code if invalid. (xxx_mbtowc, xxx_wctomb) */
-#define RET_ILSEQ      0
-/* Return code if only a shift sequence of n bytes was read. (xxx_mbtowc) */
-#define RET_TOOFEW(n)  (-1-(n))
-/* Return code if output buffer is too small. (xxx_wctomb, xxx_reset) */
-#define RET_TOOSMALL   -1
-/* Replacement character for invalid multibyte sequence or wide character. */
-#define BAD_WCHAR ((wchar_t) 0xfffd)
-#define BAD_CHAR '?'
+int utf8_mbtowc(wchar_t* pwc, const unsigned char* src, int src_len);
 
-int utf8_mbtowc(wchar_t* pwc, const unsigned char* src, int src_len)
-{
-    if (!pwc)
-        return 0;
-
-    unsigned char c = src[0];
-
-    if (c < 0x80)
-    {
-        *pwc = c;
-        return 1;
-    }
-    else if (c < 0xc2)
-    {
-        return RET_ILSEQ;
-    }
-    else if (c < 0xe0)
-    {
-        if (src_len < 2)
-            return RET_TOOFEW(0);
-        if (!((src[1] ^ 0x80) < 0x40))
-            return RET_ILSEQ;
-        *pwc = ((wchar_t)(c & 0x1f) << 6) | (wchar_t)(src[1] ^ 0x80);
-        return 2;
-    }
-    else if (c < 0xf0)
-    {
-        if (src_len < 3)
-            return RET_TOOFEW(0);
-        if (!((src[1] ^ 0x80) < 0x40 && (src[2] ^ 0x80) < 0x40 && (c >= 0xe1 || src[1] >= 0xa0)))
-            return RET_ILSEQ;
-        *pwc = ((wchar_t)(c & 0x0f) << 12) | ((wchar_t)(src[1] ^ 0x80) << 6) | (wchar_t)(src[2] ^ 0x80);
-        return 3;
-    }
-    else if (c < 0xf8)
-    {
-        if (src_len < 4)
-            return RET_TOOFEW(0);
-        if (!((src[1] ^ 0x80) < 0x40 && (src[2] ^ 0x80) < 0x40 && (src[3] ^ 0x80) < 0x40 && (c >= 0xf1 || src[1] >= 0x90)))
-            return RET_ILSEQ;
-        *pwc = ((wchar_t)(c & 0x07) << 18) | ((wchar_t)(src[1] ^ 0x80) << 12) | ((wchar_t)(src[2] ^ 0x80) << 6) | (wchar_t)(src[3] ^ 0x80);
-        return 4;
-    }
-    else if (c < 0xfc)
-    {
-        if (src_len < 5)
-            return RET_TOOFEW(0);
-        if (!((src[1] ^ 0x80) < 0x40 && (src[2] ^ 0x80) < 0x40 && (src[3] ^ 0x80) < 0x40 && (src[4] ^ 0x80) < 0x40 && (c >= 0xf9 || src[1] >= 0x88)))
-            return RET_ILSEQ;
-        *pwc = ((wchar_t)(c & 0x03) << 24) | ((wchar_t)(src[1] ^ 0x80) << 18) | ((wchar_t)(src[2] ^ 0x80) << 12) | ((wchar_t)(src[3] ^ 0x80) << 6) |
-               (wchar_t)(src[4] ^ 0x80);
-        return 5;
-    }
-    else if (c < 0xfe)
-    {
-        if (src_len < 6)
-            return RET_TOOFEW(0);
-        if (!((src[1] ^ 0x80) < 0x40 && (src[2] ^ 0x80) < 0x40 && (src[3] ^ 0x80) < 0x40 && (src[4] ^ 0x80) < 0x40 && (src[5] ^ 0x80) < 0x40 &&
-              (c >= 0xfd || src[1] >= 0x84)))
-            return RET_ILSEQ;
-        *pwc = ((wchar_t)(c & 0x01) << 30) | ((wchar_t)(src[1] ^ 0x80) << 24) | ((wchar_t)(src[2] ^ 0x80) << 18) | ((wchar_t)(src[3] ^ 0x80) << 12) |
-               ((wchar_t)(src[4] ^ 0x80) << 6) | (wchar_t)(src[5] ^ 0x80);
-        return 6;
-    }
-    else
-        return RET_ILSEQ;
-}
-
-int utf8_wctomb(unsigned char* dest, wchar_t wc, int dest_size)
-{
-    if (!dest)
-        return 0;
-
-    int count;
-    if (wc < 0x80)
-        count = 1;
-    else if (wc < 0x800)
-        count = 2;
-    else if (wc < 0x10000)
-        count = 3;
-    else if (wc < 0x200000)
-        count = 4;
-    else if (wc < 0x4000000)
-        count = 5;
-    else if (wc <= 0x7fffffff)
-        count = 6;
-    else
-        return RET_ILSEQ;
-    if (dest_size < count)
-        return RET_TOOSMALL;
-    switch (count)
-    { /* note: code falls through cases! */
-        case 6:
-            dest[5] = 0x80 | (wc & 0x3f);
-            wc = wc >> 6;
-            wc |= 0x4000000;
-        case 5:
-            dest[4] = 0x80 | (wc & 0x3f);
-            wc = wc >> 6;
-            wc |= 0x200000;
-        case 4:
-            dest[3] = 0x80 | (wc & 0x3f);
-            wc = wc >> 6;
-            wc |= 0x10000;
-        case 3:
-            dest[2] = 0x80 | (wc & 0x3f);
-            wc = wc >> 6;
-            wc |= 0x800;
-        case 2:
-            dest[1] = 0x80 | (wc & 0x3f);
-            wc = wc >> 6;
-            wc |= 0xc0;
-        case 1:
-            dest[0] = (unsigned char)wc;
-    }
-    return count;
-}
+int utf8_wctomb(unsigned char* dest, wchar_t wc, int dest_size);
 
 //////////////////////////////////////////////////
 //
 // Original - For testing
 //
 
-std::wstring utf8_mbstowcs_orig(const std::string& str)
-{
-    std::wstring wstr;
-    wchar_t      wc;
-    unsigned int sn = 0;
-    int          un = 0;
-
-    const unsigned char* s = (const unsigned char*)str.c_str();
-
-    while (sn < str.length() && *s != 0 && (un = utf8_mbtowc(&wc, s, str.length() - sn)) > 0)
-    {
-        wstr.push_back(wc);
-        s += un;
-        sn += un;
-    }
-    return wstr;
-}
-
-std::string utf8_wcstombs_orig(const std::wstring& wstr)
-{
-    std::string str;
-    char        utf8[6];
-    int         un = 0;
+std::wstring utf8_mbstowcs_orig(const std::string& str);
 
-    for (unsigned int i = 0; i < wstr.size(); ++i)
-    {
-        un = utf8_wctomb((unsigned char*)utf8, wstr[i], 6);
-        if (un > 0)
-            str.append(utf8, un);
-    }
-    return str;
-}
+std::string utf8_wcstombs_orig(const std::wstring& wstr);
 
 //////////////////////////////////////////////////
 //
-// Optimized - faster for strings smaller than SMALL_STRING_LIMIT
+// Optimized - faster for smaller strings
 //
-#define SMALL_STRING_LIMIT 1000
-
-std::wstring utf8_mbstowcs(const std::string& str)
-{
-    const unsigned char* s = (const unsigned char*)str.c_str();
-    const unsigned int   length = str.length();
-
-    if (length < SMALL_STRING_LIMIT)
-    {
-        // Faster but limited size
-        uint cCharacters = length + 1;
-        uint cBytes = (cCharacters) * sizeof(wchar_t);
-
-        wchar_t*     buffer = (wchar_t*)alloca(cBytes);
-        wchar_t*     ptr = buffer;
-        wchar_t      wc;
-        unsigned int sn = 0;
-        int          un = 0;
-
-        while (sn < length && *s != 0 && (un = utf8_mbtowc(&wc, s, length - sn)) > 0)
-        {
-            *ptr++ = wc;
-            s += un;
-            sn += un;
-        }
-        size_t usedsize = ptr - buffer;
-        return std::wstring(buffer, usedsize);
-    }
-    else
-    {
-        // Slower but any size
-        std::wstring wstr;
-        wchar_t      wc;
-        unsigned int sn = 0;
-        int          un = 0;
-
-        while (sn < length && *s != 0 && (un = utf8_mbtowc(&wc, s, length - sn)) > 0)
-        {
-            wstr.push_back(wc);
-            s += un;
-            sn += un;
-        }
-        return wstr;
-    }
-}
-
-// Optimized
-std::string utf8_wcstombs(const std::wstring& wstr)
-{
-    const unsigned int size = wstr.length();
 
-    if (size < SMALL_STRING_LIMIT)
-    {
-        // Faster but limited size
-        uint  cBytes = (size + 1) * 6;
-        char* buffer = (char*)alloca(cBytes);
-        char* ptr = buffer;
-        for (unsigned int i = 0; i < size; ++i)
-        {
-            ptr += utf8_wctomb((unsigned char*)ptr, wstr[i], 6);
-        }
-        size_t usedsize = ptr - buffer;
-        return std::string(buffer, usedsize);
-    }
-    else
-    {
-        // Slower but any size
-        char        utf8[6];
-        std::string str;
+std::wstring utf8_mbstowcs(const std::string& str);
 
-        for (unsigned int i = 0; i < size; ++i)
-        {
-            int un = utf8_wctomb((unsigned char*)utf8, wstr[i], 6);
-            if (un > 0)
-                str.append(utf8, un);
-        }
-        return str;
-    }
-}
+std::string utf8_wcstombs(const std::wstring& wstr);